PDG_WorkItem.h

Go to the documentation of this file.

/*
 * PROPRIETARY INFORMATION.  This software is proprietary to
 * Side Effects Software Inc., and is not to be reproduced,
 * transmitted, or disclosed in any way without written permission.
 *
 * COMMENTS:
 */

#ifndef __PDG_WORKITEM_H__
#define __PDG_WORKITEM_H__

#include "PDG_API.h"

#include "PDG_AttributeOwner.h"
#include "PDG_AttributeTypes.h"
#include "PDG_EventEmitter.h"
#include "PDG_EventTypes.h"
#include "PDG_File.h"
#include "PDG_LoopInfo.h"
#include "PDG_RegisteredType.h"
#include "PDG_SchedulerTypes.h"
#include "PDG_WorkItemDirty.h"
#include "PDG_WorkItemStats.h"
#include "PDG_WorkItemTypes.h"

#include <PDGE/PDGE_Dependency.h>
#include <PDGE/PDGE_DependencyOwner.h>
#include <PDGE/PDGE_Resolutions.h>
#include <PDGT/PDGT_Value.h>

#include <UT/UT_ArrayStringSet.h>
#include <UT/UT_Options.h>
#include <UT/UT_StringHolder.h>
#include <UT/UT_TBBSpinLock.h>
#include <UT/UT_WorkBuffer.h>

class PDG_AttributeEvaluator;
class PDG_BatchWorkItem;
class PDG_Graph;
class PDG_GraphContext;
struct PDG_MemoryInfo;
class PDG_Node;
class PDG_Scheduler;
class PDG_WorkItemData;

class PDG_API PDG_WorkItem : public PDG_EventEmitter,
                             public PDG_AttributeOwner,
                             public PDGE_DependencyOwner
{    
public:
    /// Default, invalid work item ID value.
    static constexpr PDG_WorkItemID theInvalidId = -1;

    /// Enumeration of pending dirty states
    enum PendingDirty : uint8
    {
        /// The item does not require any sort of dirtying
        ePendingNone,

        /// The item needs to be dirtied, but not deleted
        ePendingDirty,

        /// the item needs to be deleted
        ePendingDelete
    };

public:
    /// Constructs and initialize a new work item. This method will get a
    /// unique id for the item from the PDG_Graph, unless a valid id is 
    /// passed in
                                PDG_WorkItem(
                                    PDG_GraphContext* context,
                                    bool is_static = true,
                                    PDG_WorkItemType type = PDG_WorkItemType::eRegular,
                                    PDG_WorkItemID id = theInvalidId);

    /// Deletes this work item. Does not delete any dependencies or remove
    /// references in other items or node. The appropriate dirty method must
    /// be called first.
                                ~PDG_WorkItem() override;

    /// Returns the list of supported event types
    const PDG_EventFilter&      supportedEventTypes() const override;

    /// Returns memory usage for this work item
    int64                       getMemoryUsage(bool inclusive) const override;

    /// Returns a break down of the memory used by the work item
    virtual void                memoryInfo(PDG_MemoryInfo& memory_info,
                                           bool inclusive) const;

    /// Returns the owning node's attribute lock
    UT_RWLock*                  attribGlobalLock() const override;

    /// Returns true if the work item's attributes are writeable
    bool                        attribCanWrite(
                                    const PDG_EvaluationContext* ctx) const override;

    /// Called when the attributes have been unlocked
    void                        attribUnlock(bool modified) const override;

    /// Called when an attributed owned by the work item is modified
    void                        attribModify(
                                    PDG_AttributeType attrib_type,
                                    const UT_StringHolder& attrib_name) const override;

    /// Called when an attribute operation produces a non-fatal warning
    void                        attribWarn(
                                    const UT_StringHolder& message) const override;

    /// Returns a descriptive name of the attribute owner
    UT_StringHolder             attribOwnerName() const override;

    /// Resets the work item's dependency objects
    void                        resetOwner() override;

    /// Returns the debug name for this work item, used by PDGE for
    /// logging and debuggin features
    UT_StringHolder             debugName() const override
                                    { return name(); }

    /// Returns the debug group for this work item, used by PDGE for
    /// outputting a DOT graph.
    UT_StringHolder             debugGroup() const override;


    /// Returns the can cook dependency object for this work item
    PDGE_Dependency*            canCookDep()
                                    { return &myCanCookDep; }

    /// Returns the is cooked dependency object for this work item
    PDGE_Dependency*            isCookedDep()
                                    { return &myIsCookedDep; }

    /// Returns the work item type
    PDG_WorkItemType            type() const
                                    { return myType; }

    /// Returns the work item execution type
    PDG_WorkItemExecutionType   executionType() const
                                    { return myExecutionType; }

    /// Returns the work item cook type
    PDG_WorkItemCookType        cookType() const
                                    { return myCookType; }

    /// Returns the id of the work item, guaranteed to be unique
    PDG_WorkItemID              id() const
                                    { return myId; }


    /// Returns the name of the work item, guaranteed to be unique
    UT_StringHolder             name() const;

    /// Returns true if the work item has a command line string set, for
    /// any platform
    bool                        hasCommand() const;

    /// Returns true if the work item has platform-specific command line
    /// strings
    bool                        hasPlatformCommand() const;

    /// Returns the generic command associated with the work item
    const UT_StringHolder&      command() const;

    /// Returns the platform specific command associated with the work item,
    /// or the command for the current platform if no platform is specified
    const UT_StringHolder&      platformCommand(
                                    PDG_Platform platform=
                                        PDG_Platform::eUndefined);

    /// Returns true if the command should run in the shell
    bool                        shouldRunInShell() const;

    /// Returns the label assocaited with the work item
    UT_StringHolder             label() const;

    /// Returns true if the work item has a custom label
    bool                        hasLabel() const;

    /// Returns the custom state string associated with the work item, if
    /// one has be set.
    const UT_StringHolder&      customState() const;

    /// Returns true if the work item has a custom state string
    bool                        hasCustomState() const;


    /// Returns the node that owns the work item
    PDG_Node*                   node() const
                                    { return myNode; }

    /// Returns the context that owns the work item
    PDG_GraphContext*           context() const
                                    { return myContext; }

    /// Returns the graph that the work item belongs to
    PDG_Graph*                  graph() const;


    /// Returns the batch item that owns the work item, or nullptr if
    /// the item is not in a batch
    PDG_BatchWorkItem*          batchParent() const
                                    { return myBatchParent; }

    /// Returns the clone target for the work item, or nullptr if a specific
    /// target was not set
    const PDG_WorkItem*         cloneTarget() const
                                    { return myCloneTarget; }

    /// Const acessor to the work item's parent
    const PDG_WorkItem*         parent() const;

    /// Returns the work item that this work item copied its attributes
    /// from. If the work item is a partition, this method returns nullptr.
    const PDG_WorkItem*         attributeSource() const;


    /// Returns the work item index
    int                         index() const
                                    { return myIndex; }

    /// Returns the work item's internal index
    int                         internalIndex() const
                                    { return myInternalIndex; }

    /// Returns the work items index within its batch, or -1 if the work item
    /// is not in a batch
    int                         batchIndex() const
                                    { return myBatchIndex; }

    /// Returns the work item priority
    int                         priority() const
                                    { return myPriority; }


    /// Returns the cache id of the work item
    PDG_CacheID                 cacheId() const
                                    { return myCacheId; }

    /// Returns the work item frame
    fpreal                      frame() const
                                    { return myFrame; }

    /// Returns true if the frame has been set
    bool                        hasFrame() const
                                    { return myHasFrame; }

    /// Returns the work item step size, primarily used with batch items
    fpreal                      frameStep() const
                                    { return myFrameStep; }

    /// Returns true if the work item has any warning log messages
    bool                        hasWarnings() const
                                    { return myHasWarnings; }

    /// Returns true if the item is a static work item
    bool                        isStatic() const
                                    { return myIsStatic; }

    /// Returns true if the work item is a regular item or batch item
    bool                        isRegular() const;

    /// Returns true if the work item is a partition type
    bool                        isPartition() const;

    /// Returns true if the work item is a dynamic, regular work item
    bool                        isDynamic() const;

    /// Returns true if the work tiem is a batch
    bool                        isBatch() const;

    /// Returns true if the work item is frozen
    bool                        isFrozen() const
                                    { return myIsFrozen; }

    /// Returns true if the work item is in process
    bool                        isInProcess() const
                                    { return (myCookType ==
                                        PDG_WorkItemCookType::eInProcess); }

    /// Returns true if the work item is marked as a service task
    bool                        isServiceMode() const
                                    { return (myCookType ==
                                        PDG_WorkItemCookType::eService); }

    /// Returns true if the work item is scheduled out of process
    bool                        isOutOfProcess() const
                                    { return (myCookType ==
                                        PDG_WorkItemCookType::eOutOfProcess); }

    /// Returns true if the work item is scheduled (either service, in, or out
    /// of process).
    bool                        isScheduled() const
                                    {
                                        if (myCookType ==
                                            PDG_WorkItemCookType::eGenerate)
                                        {
                                            return false;
                                        }

                                        if (myCookType ==
                                            PDG_WorkItemCookType::eAutomatic)
                                        {
                                            return false;
                                        }

                                        return true;
                                    }

    /// Returns true if the work item has post-cook logic
    bool                        isPostCook() const
                                    { return myIsPostCook; }

    /// Returns true if the work item is flagged as not being allowed to
    /// generate child items.
    bool                        isNoGenerate() const
                                    { return myIsNoGenerate; }

    /// Returns true if the work item is marked as needing to clone output
    /// files from its parent
    bool                        isCloneOutputFiles() const
                                    { return myIsCloneOutputFiles; }

    /// Returns true if the work item has to cook on the main thread, when
    /// it cooks in process. This is determined by the script info from
    /// the owning node
    bool                        isMainThread() const;

    /// Returns the work item cook state
    PDG_WorkItemState           state() const
                                    { return myState; }

    /// Returns the worst dependency state, e.g. if any of the deps have failed
    /// or canceled.
    PDG_WorkItemState           dependencyState() const
                                    { return myDepState; }

    /// Returns true if the work item is in a cooked state
    bool                        isCooked() const;

    /// Returns true if the work item is in a successful cooked state
    bool                        isSuccessful() const;

    /// Returns true if the work item is in a unsucessful cooked state
    bool                        isUnsuccessful() const;

    /// Returns true if the work item is uncooked
    bool                        isUncooked() const;

    /// Returns true if the work item has an unsucessful dependency
    bool                        isUnsuccessfulDep() const;

    /// Returns true if the work item is an cooking/scheduled state
    bool                        isProcessing() const;

    /// Returns true if the work item is long running
    bool                        isLongRunning() const;

    /// Returns an enum indicating if work item is ready to cook, blocked, or
    /// has failed dependencies
    PDG_WorkItemEvalState       evaluationState() const;


    /// Returns the edge lock for the work item
    UT_TBBSpinLock&             edgeLock() const
                                    { return myEdgeLock; }

    /// Returns the dependencies, e.g. work items that this item depends on. Not
    /// thread safe.
    const PDG_WorkItemIDMap&    dependenciesUnsafe() const;

    /// Returns the dependents, e.g. work items that depend on this item. Not
    /// thread safe.
    const PDG_WorkItemIDMap&    dependentsUnsafe() const;

    /// Returns a copy of the dependencies in an array
    void                        dependencies(
                                    PDG_WorkItemConstArray& dependency_array) const; 

   /// Returns the dependency ids in a set
    void                        dependencyIds(
                                    PDG_WorkItemIDSet& dependency_ids) const;

    /// Returns the number of dependencies
    int                         dependencySize() const;

    /// Returns a copy of the dependents in an array
    void                        dependents(
                                    PDG_WorkItemConstArray& dependent_array) const; 

    /// Returns the dependent ids in a set
    void                        dependentIds(
                                    PDG_WorkItemIDSet& dependent_ids) const;

    /// Returns the root failed dependencies for this work item. This doesn't
    /// include intermediate tasks in the dep chain that also failed because of
    /// the same upstream failure
    void                        failedDependencies(
                                    PDG_WorkItemConstArray& dependecies) const;

    /// Returns a copy of the "visual" dependencies, e.g. the ones the user
    /// would see if they used TOPs or wanted to print out the DAG itself. This
    /// strips out any intenal items/dependencies used by TOPs
    void                        visualDependencies(
                                    PDG_WorkItemSet& deps,
                                    bool expand,
                                    PDG_Scheduler* filter) const;

    /// Inverse of the above
    void                        visualDependents(
                                    PDG_WorkItemSet& deps,
                                    bool expand,
                                    PDG_Scheduler* filter) const;

    /// Returns the require dependencies, e.g. deps that own this item. This is
    /// used with partitioning to force the partition to by delete when one of
    /// it's required dependencies is deleted
    const PDG_WorkItemIDBoolMap&
                                requiredDependencies() const;

    /// Returns true if the item is a partition, and stores the partition
    /// components into the array passed in as a parameter
    bool                        partitionItems(
                                    PDG_WorkItemConstArray& components) const;

    /// Flattens dependencies by query all dependencies of the work item,
    /// and if the the dependency is a wrapper expands it into real items that
    /// the wrapper depends on.
    void                        sortedDependencies(
                                    PDG_WorkItemConstArray& sorted_depencies,
                                    bool only_upstream=false) const;

    /// Adds a dependency to the work item, and a corresponding dependent
    /// back pointer to the incoming item.
    ///
    /// Returns false if the dependency already exists, and returns true if the
    /// dependency does not
    bool                        addDependency(
                                    PDG_WorkItem* work_item,
                                    bool required = false,
                                    bool user_defined = false);

    /// Marks an existing dependency as a required dep. The work item passed in
    /// *must* already be a dependency
    void                        addRequiredDependency(
                                    const PDG_WorkItem* work_item,
                                    bool user_defined);

    /// Checks if the work item has a given item as a dependency
    bool                        hasDependency(
                                    const PDG_WorkItem* work_item) const;

    /// Removes a dependency and the corresponding back pointer from the target
    /// work item.
    bool                        removeDependency(PDG_WorkItem* work_item);

    /// Updates the loop stack, with the specified parent
    void                        pushLoopInfoStack(
                                    const PDG_WorkItem* parent_item);

    /// Sets the loop stack contents
    void                        setLoopInfoStack(
                                    const PDG_LoopInfo::Stack& loop_stack);

    /// Returns the loop stack entry at the specified index;
    PDG_WorkItem*               loopInfoStackEntry(int index) const;

    /// Returns the loop iteration of the work item
    int                         loopInfoIteration() const;

    /// Returns the loop number of the work item
    int                         loopInfoNumber() const;

    /// Returns the size of the loop that contains the work item
    int                         loopInfoSize() const;

    /// Returns the loop depth of the work item
    int                         loopInfoDepth() const;

    /// Returns the loop service lock ID of the work item
    PDG_WorkItemID              loopInfoLock() const;

    /// Returns the loop stack for this work item
    const PDG_LoopInfo::Stack&  loopInfoStack() const
                                    { return myLoopInfoStack; }

    /// Compares the loop info stack of another work item with this one,
    /// returning true if they put in the same partition or false if they're
    /// from independent loops.
    bool                        compareLoopInfoStack(
                                    const PDG_WorkItem* other) const;

    /// Sets the is post cook flag
    void                        setIsPostCook(bool postcook);

    /// Sets the work item no generate flag
    void                        setIsNoGenerate(bool no_gen);

    /// Sets the work item's clone output files flag
    void                        setIsCloneOutputFiles(bool clone);

    /// Sets the work item state
    bool                        setState(
                                    PDG_WorkItemState state,
                                    bool emit=true,
                                    bool update_node=true);

    /// Sets the frozen flag on the work item
    void                        setIsFrozen(bool is_frozen);

    /// Sets the execution type of the work item
    void                        setExecutionType(
                                    PDG_WorkItemExecutionType execution_type);

    /// Sets the cook type for the work item
    void                        setCookType(PDG_WorkItemCookType cook_type);

    /// Returns the work item's modified flag
    bool                        wasModified() const
                                    { return myWasModified; }

    /// Sets the work item index
    void                        setIndex(int index);

    /// Sets the work items internal index, used for dirtying
    void                        setInternalIndex(int index)
                                    { myInternalIndex = index; }

    /// Sets the work item priority
    void                        setPriority(int priority, bool emit=true);

    /// Boosts the priority of the work item and its dependencies, if they
    /// have not yet started to cook
    void                        boostPriority();

    /// Sets the work item frame. This methods takes a step size since it gets
    /// overridden by batch work items to initialize the frame of each sub item
    /// as frame + index*step_size;
    bool                        clearFrame();
    bool                        setFrame(fpreal frame);
    bool                        setFrame(fpreal frame, fpreal frame_step);

    /// Sets the work item node
    void                        setNode(PDG_Node* node, bool emit);

    /// Sets the pending dirty flag on the work item, used for regeneration
    virtual void                setPendingDirty(PendingDirty pending);

    /// Called when the work item should begin regenerating
    void                        startRegenerating();

    /// Called when the work item is finished regenerating, in order to
    /// determine if it needs to be deleted or dirtied
    PendingDirty                doneRegenerating();

    /// Sets the clone target, e.g. the item from which this item will
    /// clone attributes
    void                        setCloneTarget(
                                    const PDG_WorkItem* clone_target);

    /// Sets the batch item info, e.g. the batch item pointer and the index
    /// within the batch.
    void                        setBatchInfo(PDG_BatchWorkItem* item,
                                             int index,
                                             int batch_index,
                                             int priority);

    /// Updates the loop iteration and number, at the head of the current
    /// loop stack
    void                        setLoopInfo(
                                    int iteration,
                                    int number,
                                    int size,
                                    PDG_WorkItemID service_lock=theInvalidId);

    /// Sets the work item command directly
    virtual bool                setCommand(
                                    const UT_StringHolder& command,
                                    bool shell=false);

    /// Sets a per-platform work item command
    virtual bool                setPlatformCommands(
                                    const UT_StringHolder& linux,
                                    const UT_StringHolder& mac,
                                    const UT_StringHolder& windows,
                                    bool shell=false);

    /// Sets the work item label
    void                        setLabel(const UT_StringHolder& label);

    /// Clears the work item label
    void                        clearLabel();

    /// Sets the custom state string. Note that custom states are cleared
    /// any time the actual work item state changes.
    void                        setCustomState(const UT_StringHolder& state);

    /// Clears the custom state string
    void                        clearCustomState();

    bool                        hasCookPercent() const;
    fpreal                      cookPercent() const;
    void                        setCookPercent(const fpreal &cook_percent);

    /// Appends a formatted error message to the work item's in process
    /// log buffer
    template <typename... Args>
    void                        addErrorFmt(
                                        const char* fmt,
                                        Args&&... args) const
                                    {
                                        appendLogFmt(
                                            true,
                                            0,
                                            "ERROR",
                                            fmt,
                                            std::forward<Args>(args)...); 
                                    }

    /// Appends a plain error message to the work item's in process
    /// log buffer
    void                        addError(
                                        const UT_StringHolder& log,
                                        bool timestamp=true) const
                                    {
                                        appendLogPrefix(
                                            timestamp,
                                            0,
                                            "ERROR",
                                            log);
                                    }

    /// Appends a formatted warning message to the work item's in process
    /// log buffer
    template <typename... Args>
    void                        addWarningFmt(
                                        const char* fmt,
                                        Args&&... args) const
                                    {
                                        appendLogFmt(
                                            true,
                                            0,
                                            "WARNING",
                                            fmt,
                                            std::forward<Args>(args)...); 
                                        myHasWarnings = true;
                                    }

    /// Appends a plain warning message to the work item's in process
    /// log buffer
    void                        addWarning(
                                        const UT_StringHolder& log,
                                        bool timestamp=true,
                                        int verbosity=0) const
                                    {
                                        appendLogPrefix(
                                            timestamp,
                                            verbosity,
                                            "WARNING",
                                            log);
                                        myHasWarnings = true;
                                    }

    /// Appends a formatted message to the work item's in process
    /// log buffer
    template <typename... Args>
    void                        addMessageFmt(
                                        const char* fmt,
                                        Args&&... args) const
                                    {
                                        appendLogFmt(
                                            true,
                                            0,
                                            UT_StringHolder::theEmptyString,
                                            fmt,
                                            std::forward<Args>(args)...); 
                                    }

    /// Appends a plain message to the work item's in process log buffer
    void                        addMessage(
                                        const UT_StringHolder& log,
                                        bool timestamp=true,
                                        int verbosity=0) const
                                    {
                                        appendLogPrefix(
                                            timestamp,
                                            verbosity,
                                            UT_StringHolder::theEmptyString,
                                            log);
                                    }

    /// Appends text data to the log with the given log type
    void                        addLog(
                                    PDG_WorkItemLogType type,
                                    const UT_StringHolder& log,
                                    bool timestamp=true,
                                    int verbosity=0) const;

    /// Returns the current in process log buffer
    const UT_WorkBuffer&        logMessages() const
                                    { return myLogMessages; }

    /// Returns the application path to the work item, e.g. /node/workitem
    void                        appPath(UT_WorkBuffer& buffer) const;

    /// Dirties the work item by initiating a dirty operation from the graph.
    /// Probably the method you want to call to dirty the item if you aren't
    /// working in $PDG directly
    void                        dirty(bool should_delete, bool remove_outputs);

    /// Prepares the work item for dirtying, but does not actually modify the
    /// item or its dependencies in any way.
    PDG_WorkItemDirty           prepareDirty(bool should_delete);

    /// Cancels the work item's execution, if it's running with a scheduler
    void                        cancel();

    /// Freezes the attributes of the work item and clears out all references
    /// to upstream work items and data.
    void                        freeze();

    /// Invalidates the file cache of this work item
    void                        invalidateCache();

    /// Syncs work item data with its parent
    void                        syncData(bool force_recursive);

    /// Getter/setter for the backing work item data object, which might be a
    /// user defined class
    PDG_WorkItemData*           data() const;
    void                        setData(PDG_WorkItemDataPtr data);
    bool                        compareDataType(
                                            const PDG_BaseType* other_type,
                                            bool deep=true) const;
    SYS_HashType                hash() const;

    /// Does a numeric data lookup, for use with the @ operator
    PDG_AttributeCast           numericData(
                                        fpreal& param,
                                        exint& query_index,
                                        const PDG_AttributeEvaluator& evaluator,
                                        fpreal frame,
                                        int thread) const;

    /// Does a string data lookup, for use with the @ operator
    PDG_AttributeCast           stringData(
                                        UT_WorkBuffer& buffer,
                                        exint& query_index,
                                        const PDG_AttributeEvaluator& evaluator,
                                        fpreal frame,
                                        int thread) const;

    /// Serialization of underlying PDG_WorkItemData object
    UT_StringHolder             serializeData() const;
    bool                        serializeDataToFile(
                                        const UT_StringHolder& file) const;

    /// Compares the work item with another item
    bool                        compare(PDG_WorkItem* item) const;

   
    /// Runs selection logic from selecting a work item in TOPs
    bool                        uiSelect(bool select);


    /// Returns true if the work item has at least one output file
    bool                        hasOutputFiles() const;

    /// Returns the output file cache state of this work item
    virtual PDG_WorkItemCacheState
                                outputCacheState() const;

    /// Returns the raw output file array for the work item. The paths in this
    /// array are unlocalized, i.e. they may contain __PDG_DIR__
    bool                        outputFiles(
                                        PDG_File::Array& files) const;

    /// Localizes output files and returns it back in the supplied array.
    /// Returns false if the work item has no node or the node has no
    /// scheduler.
    bool                        localizedOutputFiles(
                                        PDG_File::Array& files) const;

    /// Returns the local temporary file directory from the sechduler
    /// associated with this work item.
    UT_StringHolder             tempDir() const;

    /// Returns all output files that are a substring-match for the given tag
    bool                        outputFilesForTag(
                                        PDG_File::Array& files,
                                        const UT_StringHolder& tag,
                                        bool include_expected = false) const;

    /// Localizes a file path with the scheduler associated with the work
    /// item
    UT_StringHolder             localizePath(
                                        const UT_StringHolder& path) const;

    /// Returns the first output file that is a substring-match for the given
    /// tag
    UT_StringHolder             firstOutputFileForTag(
                                        const UT_StringHolder& tag) const;

    /// Returns the list of expected output files
    bool                        expectedOutputFiles(
                                        PDG_File::Array& files) const;

    /// Adds an expected output file to the work item
    void                        addExpectedOutputFile(
                                            const UT_StringHolder& path,
                                            const UT_StringHolder& tag,
                                            bool own);

    /// Adds an array of expected output file paths to the work item
    void                        addExpectedOutputFiles(
                                            const UT_StringArray& paths,
                                            const UT_StringHolder& tag,
                                            bool own);

    /// Adds an expected output file to the work item
    void                        addExpectedOutputFile(const PDG_File& file);

    /// Appends the given output file to the work item
    void                        addOutputFile(
                                            const UT_StringHolder& path,
                                            const UT_StringHolder& tag,
                                            PDG_File::Hash hash_code,
                                            bool own);

    /// Appends the given array of output files to the work item. If the
    /// length of the hashes array is less than the paths, a default value
    /// of 0 will be used for paths without a corresponding hash code. I.e
    /// an empty hash array can be passed in to use 0 for each file. Assumes
    /// that the same tag is used for all work items
    void                        addOutputFiles(
                                            const UT_StringArray& paths,
                                            const UT_StringHolder& tag,
                                            const PDG_File::HashArray& hashes,
                                            bool own);

    /// Appends the given array of output files to the work item. If the length
    /// of the hashes or tags array is less than the paths, a default value of
    /// 0 and "" are used respectively.
    void                        addOutputFiles(
                                            const UT_StringArray& paths,
                                            const UT_StringArray& tags,
                                            const PDG_File::HashArray& hashes,
                                            bool own);

    /// Replaces the output file at the given index, if that index is valid
    void                        updateOutputFile(
                                            int index, 
                                            const UT_StringHolder& path, 
                                            const UT_StringHolder& tag, 
                                            PDG_File::Hash hash_code,
                                            bool owner);

    /// Returns the input files for immediate dependencies
    void                        inputFiles(
                                        PDG_File::Array& files,
                                        bool include_expected) const;

    /// Returns all input files that are a substring-match for the given tag
    bool                        inputFilesForTag(
                                        PDG_File::Array& files,
                                        const UT_StringHolder& tag,
                                        bool include_expected = false) const;

    /// Checks if the work items output files are valid, e.g. they exist
    bool                        validateOutputFiles();

    /// Transfers all dependent files to the remote directory, using the
    /// supplied scheduler
    bool                        transferFiles(bool update_cache);

    /// Clears all output files
    void                        clearOutputFiles();

    /// Clears expected outputs
    void                        clearExpectedOutputFiles();

    /// Clears output files added during the cook. Note that this method is
    /// only safe to call if the item has not yet been marked as cooked.
    void                        clearRuntimeOutputFiles();

    /// Sets the clear environment flagging, effectively stopping any upstream
    /// env vars from being considered beyond this work item
    void                        clearEnvironment();

    /// Adds an environment variable and value to this work items env var map
    bool                        addEnvironmentVar(
                                        const UT_StringHolder& var_name,
                                        const PDGT_Value& value,
                                        UT_WorkBuffer& errors);

    /// Returns true if the work item has the specified env var
    bool                        hasEnvironmentVar(
                                        const UT_StringHolder& var_name) const;

    /// Globs the full set of environment variables for this work item by
    /// searching through it's parents.
    void                        globEnvironment(
                                        PDGT_Value::Map& environment) const;

    /// Looks up a key in the environment, returns true if the key exists
    bool                        envLookup(const char* key,
                                          PDGT_Value& value) const;

    /// Adds a warning to the owning node
    void                        addNodeWarning(
                                    const UT_StringHolder& message) const;

    /// Starts cooking a sub item in a batch, and optionally waits for it to
    /// be available. Returns false if the item cannot start due to a failure.
    bool                        startSubItem(bool wait, UT_WorkBuffer& errors);

    /// Returns true if the sub item is ready
    bool                        checkSubItem(UT_WorkBuffer& errors);

    /// Marks a work item is cooking, updates it cook start time, and prints
    /// log messages.
    bool                        startWorkItem();

    /// Marks the work item as cooked, if it is an in-process batch item
    bool                        cookSubItem(
                                    PDG_WorkItemState state,
                                    fpreal duration);

    /// Returns the scheduler associated with the work item, looked up from the
    /// work item's node.
    PDG_Scheduler*              scheduler() const;

    /// Returns true if the work item is schduled by the specified schduler
    bool                        isScheduledBy(PDG_Scheduler* scheduler) const;

    /// Reports a performance monitor event for this work item
    void                        performanceEvent() const;

    /// Sets a stat to the current time
    virtual fpreal              setTimeStat(PDG_WorkItemStats::StatType stat,
                                            bool batched);

    /// Returns the perf stats object for the work item
    const PDG_WorkItemStats&    stats() const
                                    { return myStats; }
    PDG_WorkItemStats&          stats()
                                    { return myStats; }

    /// Replaces env vars in the target string with respect to the environment
    /// and properties of this work item. If py_replace is true, replaces the
    /// env vars with a valid python token instead of 
    void                        varReplace(const char* input,
                                           UT_WorkBuffer& output) const;

    /// Saves the work item to a .json file or string
    bool                        saveJSONFile(
                                    const UT_StringHolder& path,
                                    bool pretty_print,
                                    bool global_attributes,
                                    bool skip_defaults) const;
    UT_StringHolder             saveJSONString(
                                    bool pretty_print,
                                    bool global_attributes,
                                    bool skip_defaults) const;

    /// Saves the work item to a UT_OptionsHolder
    UT_OptionsHolder            saveDict(
                                    bool global_attributes,
                                    bool skip_defaults) const;

    /// Saves the array of work items to a .json file or string
    static bool                 saveArrayJSONFile(
                                    const UT_StringHolder& path,
                                    const PDG_WorkItemArray& items,
                                    bool pretty_print,
                                    bool global_attributes,
                                    bool skip_defaults);
    static UT_StringHolder      saveArrayJSONString(
                                    const PDG_WorkItemArray& items,
                                    bool pretty_print,
                                    bool global_attributes,
                                    bool skip_defaults);

    /// Saves the list of work items to a UT_OptionsHolder
    static UT_OptionsHolder     saveArrayDict(
                                    const PDG_WorkItemArray& items,
                                    bool global_attributes,
                                    bool skip_defaults);

    /// Loads a work item from a .json file or string
    static PDG_WorkItem*        loadJSONFile(
                                    const UT_StringHolder& path,
                                    bool make_active,
                                    UT_WorkBuffer& errors);
    static PDG_WorkItem*        loadJSONString(
                                    const UT_StringHolder& data,
                                    bool make_active,
                                    UT_WorkBuffer& errors);

    /// Creates a JSON patch from the work item as a string
    UT_StringHolder             createJSONPatch(bool pretty_print) const;

protected:
    friend class PDG_BatchWorkItem;
    friend class PDG_Scheduler;
    friend class PDG_WorkItemHolder;
    friend class PDG_WorkItemDirty;

    /// Dirties or deletes the work item and propogates the necessary changes
    /// to downstream items and dependents.
    void                        prepareDirty(PDG_WorkItemDirty& dirty_state,
                                             bool is_delete,
                                             bool is_dirty_data) const;

    /// Applies a dirty operation that was stored to the dirty state
    void                        applyDirty(
                                    const PDG_WorkItemDirty& dirty_state,
                                    bool remove_outputs);

    /// Runs custom dirty handlers and deletes output files
    void                        handleDirty(
                                    const UT_ArrayStringSet& keep_set,
                                    bool should_delete,
                                    bool remove_outputs);

    /// Returns the list of files owned by this work item
    void                        ownedFiles(
                                    PDG_File::Set& files,
                                    bool include_expected) const;

    /// Updates the file cache of this work item so the ID is at least as
    /// large as the input ID
    void                        updateCacheId(PDG_CacheID cache_id);


    /// Called when one of the dependencies owned by this work item
    /// is resolved.
    PDGE_Dependency::State      evalResolve(
                                        PDGE_Resolutions& resolutions,
                                        const PDGE_Evaluator& evaluator,
                                        PDGE_Dependency* dependency) override;

    /// Called when one of the dependencies owned by this work item
    /// is unresolved;
    int                         evalUnresolve(
                                        PDGE_Dependency::Array& extras,
                                        PDGE_Dependency* dependency) override;

    /// Called when an item is about to be scheduled, to check if it has 
    /// cached outputs on disk. Returns Uncooked if the item is not cached
    /// and should cook, Failure if the read files mode fails, or CookedCache
    /// if the item is cooked from cache.
    virtual PDG_WorkItemState   checkCached(bool add_outputs);

    /// Upgrades expected outputs to runtime outputs for this work item
    bool                        upgradeExpectedOutputFiles(
                                        bool check_disk,
                                        bool clear);

    /// Called by the scheduler to mark out of process items as cooked
    void                        postCook(PDGE_Resolutions& resolutions,
                                         PDG_WorkItemState result,
                                         int batch_index,
                                         fpreal duration,
                                         bool notify_all,
                                         bool static_cook,
                                         bool unresolve);

    /// Notifies this work item that is has cooked.
    virtual void                notifyCook(PDGE_Resolutions& resolutions,
                                           PDG_WorkItemState result,
                                           int batch_index,
                                           fpreal duration,
                                           bool notify_all,
                                           bool static_cook,
                                           bool unresolve);

    /// Syncs a work item's cache with its dependents
    virtual void                syncCacheId();

    /// Does an inprocess cook of the work item, returning true on success
    bool                        cook();

    /// Internal method for setting the frame
    virtual bool                setFrame(fpreal frame,
                                         fpreal frame_step,
                                         bool check_dirty);

    /// Adds a parent and copies attributes by inheriting them from the parent
    /// of another work item. This is used for generating dynamic sub items
    /// in a batch, and copying the parent information from the last batch
    /// item onto the new item
    void                        inheritParent(const PDG_WorkItem* work_item);

private:
    /// Removes all dependencies from the work item, and removes them work item
    /// from the dependents list of all dependencies
    void                        removeAllDependencies();

    /// Updates the dependency state, given the incoming state
    void                        updateDependencyState(
                                    PDG_WorkItemState dep_state);

    /// Returns a copy of the dependencies in an array, and filters out
    /// work items from the target node
    void                        upstreamDependencies(
                                    PDG_WorkItemConstArray& dependency_array) const;

    /// Helper method to query the list of failed upstream dependencies
    void                        failedDependencies(
                                    PDG_WorkItemConstArray& dependencies,
                                    PDG_WorkItemIDSet& visited) const;

    /// Called before the work item is scheduled
    bool                        prepareTask();

    /// Called when a work item should be scheduled for execution
    PDG_WorkItemState           schedule();

    /// Appends a log message with a time stamp and a new line
    void                        appendLogPrefix(
                                    bool timestamp,
                                    int verbosity,
                                    const UT_StringHolder& prefix,
                                    const UT_StringHolder& msg) const;

    /// Appends a format string to the work item's log
    template <typename... Args>
    void                        appendLogFmt(
                                        bool timestamp,
                                        int verbosity,
                                        const UT_StringHolder& prefix,
                                        const char* fmt,
                                        Args&&... args) const
                                    {
                                        UT_WorkBuffer message;
                                        message.format(
                                            fmt, std::forward<Args>(args)...); 
                                        appendLogPrefix(
                                            timestamp,
                                            verbosity,
                                            prefix,
                                            message.buffer());
                                    }

    /// Processes this work item as a priority boost dep
    void                        processBoostDependency(PDG_WorkItemSet& deps);

    /// Gathers the input dependencies from priority boosting
    void                        gatherBoostDependencies(PDG_WorkItemSet& deps);

protected:
    PDGE_Dependency             myCanCookDep;
    PDGE_Dependency             myIsCookedDep;

    PDG_WorkItemStats           myStats;

    PDG_WorkItemIDMap           myDependencies;
    PDG_WorkItemIDMap           myDependents;
    PDG_WorkItemIDBoolMap       myRequiredDependencies;

    PDG_LoopInfo::Stack         myLoopInfoStack;

    mutable UT_WorkBuffer       myLogMessages;

    PDG_Node*                   myNode;
    PDG_GraphContext*           myContext;

    PDG_BatchWorkItem*          myBatchParent;
    const PDG_WorkItem*         myCloneTarget;

    PDG_WorkItemDataPtr         myData;

    fpreal                      myFrame;
    fpreal                      myFrameStep;

    PDG_WorkItemID              myId;
    int                         myIndex;
    int                         myBatchIndex;
    int                         myPriority;
    int                         myInternalIndex;
    PDG_CacheID                 myCacheId;

    PDG_WorkItemType            myType;
    PDG_WorkItemExecutionType   myExecutionType;
    PDG_WorkItemCookType        myCookType;
    PendingDirty                myPendingDirty;
    PDG_WorkItemState           myState;
    PDG_WorkItemState           myDepState;

    mutable UT_TBBSpinLock      myEdgeLock;

    bool                        myHasFrame:1;
    mutable bool                myHasWarnings:1;

    bool                        myAttributesDirty:1;
    bool                        myWasModified:1;
    bool                        myDidCook:1;

    bool                        myIsPostCook:1;
    bool                        myIsStatic:1;
    bool                        myIsNoGenerate:1;
    bool                        myIsCloneOutputFiles:1;
    bool                        myIsFrozen:1;
    bool                        myIsBatchReady:1;
};

#endif /* __PDG_WORKITEM_H__ */