prefect_databricks.models.jobs

Classes

AccessControlList

Bases: BaseModel

See source code for the fields' description.

Source code in prefect_databricks/models/jobs.py

class AccessControlList(BaseModel):
    """
    See source code for the fields' description.
    """

    class Config:
        extra = Extra.allow
        allow_mutation = False

    access_control_list: Optional[List[AccessControlRequest]] = Field(
        None, description="List of permissions to set on the job."
    )

AccessControlRequest

Bases: AccessControlRequestForUser, AccessControlRequestForGroup

See source code for the fields' description.

Source code in prefect_databricks/models/jobs.py

class AccessControlRequest(AccessControlRequestForUser, AccessControlRequestForGroup):
    """
    See source code for the fields' description.
    """

    class Config:
        extra = Extra.allow
        allow_mutation = False

AccessControlRequestForGroup

Bases: BaseModel

See source code for the fields' description.

Source code in prefect_databricks/models/jobs.py

class AccessControlRequestForGroup(BaseModel):
    """
    See source code for the fields' description.
    """

    class Config:
        extra = Extra.allow
        allow_mutation = False

    group_name: Optional[GroupName] = None
    permission_level: Optional[PermissionLevelForGroup] = None

AccessControlRequestForServicePrincipal

Bases: BaseModel

See source code for the fields' description.

Source code in prefect_databricks/models/jobs.py

class AccessControlRequestForServicePrincipal(BaseModel):
    """
    See source code for the fields' description.
    """

    class Config:
        extra = Extra.allow
        allow_mutation = False

    permission_level: Optional[PermissionLevel] = None
    service_principal_name: Optional[ServicePrincipalName] = None

AccessControlRequestForUser

Bases: BaseModel

See source code for the fields' description.

Source code in prefect_databricks/models/jobs.py

class AccessControlRequestForUser(BaseModel):
    """
    See source code for the fields' description.
    """

    class Config:
        extra = Extra.allow
        allow_mutation = False

    permission_level: Optional[PermissionLevel] = None
    user_name: Optional[UserName] = None

AutoScale

Bases: BaseModel

See source code for the fields' description.

Source code in prefect_databricks/models/jobs.py

class AutoScale(BaseModel):
    """
    See source code for the fields' description.
    """

    class Config:
        extra = Extra.allow
        allow_mutation = False

    max_workers: Optional[int] = Field(
        None,
        description=(
            "The maximum number of workers to which the cluster can scale up when"
            " overloaded. max_workers must be strictly greater than min_workers."
        ),
    )
    min_workers: Optional[int] = Field(
        None,
        description=(
            "The minimum number of workers to which the cluster can scale down when"
            " underutilized. It is also the initial number of workers the cluster has"
            " after creation."
        ),
    )

AwsAttributes

Bases: BaseModel

See source code for the fields' description.

Source code in prefect_databricks/models/jobs.py

class AwsAttributes(BaseModel):
    """
    See source code for the fields' description.
    """

    class Config:
        extra = Extra.allow
        allow_mutation = False

    availability: Optional[Literal["SPOT", "ON_DEMAND", "SPOT_WITH_FALLBACK"]] = Field(
        None,
        description=(
            "Availability type used for all subsequent nodes past the `first_on_demand`"
            " ones. **Note:** If `first_on_demand` is zero, this availability type is"
            " used for the entire cluster.\n\n`SPOT`: use spot instances.\n`ON_DEMAND`:"
            " use on-demand instances.\n`SPOT_WITH_FALLBACK`: preferably use spot"
            " instances, but fall back to on-demand instances if spot instances cannot"
            " be acquired (for example, if AWS spot prices are too high)."
        ),
    )
    ebs_volume_count: Optional[int] = Field(
        None,
        description=(
            "The number of volumes launched for each instance. You can choose up to 10"
            " volumes. This feature is only enabled for supported node types. Legacy"
            " node types cannot specify custom EBS volumes. For node types with no"
            " instance store, at least one EBS volume needs to be specified; otherwise,"
            " cluster creation fails.\n\nThese EBS volumes are mounted at `/ebs0`,"
            " `/ebs1`, and etc. Instance store volumes are mounted at `/local_disk0`,"
            " `/local_disk1`, and etc.\n\nIf EBS volumes are attached, Databricks"
            " configures Spark to use only the EBS volumes for scratch storage because"
            " heterogeneously sized scratch devices can lead to inefficient disk"
            " utilization. If no EBS volumes are attached, Databricks configures Spark"
            " to use instance store volumes.\n\nIf EBS volumes are specified, then the"
            " Spark configuration `spark.local.dir` is overridden."
        ),
    )
    ebs_volume_iops: Optional[int] = Field(
        None,
        description=(
            "The number of IOPS per EBS gp3 volume.\n\nThis value must be between 3000"
            " and 16000.\n\nThe value of IOPS and throughput is calculated based on AWS"
            " documentation to match the maximum performance of a gp2 volume with the"
            " same volume size.\n\nFor more information, see the [EBS volume limit"
            " calculator](https://github.com/awslabs/aws-support-tools/tree/master/EBS/VolumeLimitCalculator)."
        ),
    )
    ebs_volume_size: Optional[int] = Field(
        None,
        description=(
            "The size of each EBS volume (in GiB) launched for each instance. For"
            " general purpose SSD, this value must be within the range 100 - 4096\\."
            " For throughput optimized HDD, this value must be within the range 500 -"
            " 4096\\. Custom EBS volumes cannot be specified for the legacy node types"
            " (_memory-optimized_ and _compute-optimized_)."
        ),
    )
    ebs_volume_throughput: Optional[int] = Field(
        None,
        description=(
            "The throughput per EBS gp3 volume, in MiB per second.\n\nThis value must"
            " be between 125 and 1000."
        ),
    )
    ebs_volume_type: Optional[
        Literal["GENERAL_PURPOSE_SSD", "THROUGHPUT_OPTIMIZED_HDD"]
    ] = Field(
        None,
        description=(
            "The type of EBS volume that is launched with this"
            " cluster.\n\n`GENERAL_PURPOSE_SSD`: provision extra storage using AWS gp2"
            " EBS volumes.\n`THROUGHPUT_OPTIMIZED_HDD`: provision extra storage using"
            " AWS st1 volumes."
        ),
    )
    first_on_demand: Optional[int] = Field(
        None,
        description=(
            "The first first_on_demand nodes of the cluster are placed on on-demand"
            " instances. If this value is greater than 0, the cluster driver node is"
            " placed on an on-demand instance. If this value is greater than or equal"
            " to the current cluster size, all nodes are placed on on-demand instances."
            " If this value is less than the current cluster size, first_on_demand"
            " nodes are placed on on-demand instances and the remainder are placed on"
            " `availability` instances. This value does not affect cluster size and"
            " cannot be mutated over the lifetime of a cluster."
        ),
    )
    instance_profile_arn: Optional[str] = Field(
        None,
        description=(
            "Nodes for this cluster are only be placed on AWS instances with this"
            " instance profile. If omitted, nodes are placed on instances without an"
            " instance profile. The instance profile must have previously been added to"
            " the Databricks environment by an account administrator.\n\nThis feature"
            " may only be available to certain customer plans."
        ),
    )
    spot_bid_price_percent: Optional[int] = Field(
        None,
        description=(
            "The max price for AWS spot instances, as a percentage of the corresponding"
            " instance type’s on-demand price. For example, if this field is set to 50,"
            " and the cluster needs a new `i3.xlarge` spot instance, then the max price"
            " is half of the price of on-demand `i3.xlarge` instances. Similarly, if"
            " this field is set to 200, the max price is twice the price of on-demand"
            " `i3.xlarge` instances. If not specified, the default value is 100\\. When"
            " spot instances are requested for this cluster, only spot instances whose"
            " max price percentage matches this field is considered. For safety, we"
            " enforce this field to be no more than 10000."
        ),
    )
    zone_id: Optional[str] = Field(
        None,
        description=(
            "Identifier for the availability zone/datacenter in which the cluster"
            " resides. You have three options:\n\n**Specify an availability zone as a"
            " string**, for example: “us-west-2a”. The provided availability zone must"
            " be in the same region as the Databricks deployment. For example,"
            " “us-west-2a” is not a valid zone ID if the Databricks deployment resides"
            " in the “us-east-1” region.\n\n**Enable automatic availability zone"
            " selection (“Auto-AZ”)**, by setting the value “auto”. Databricks selects"
            " the AZ based on available IPs in the workspace subnets and retries in"
            " other availability zones if AWS returns insufficient capacity"
            " errors.\n\n**Do not specify a value**. If not specified, a default zone"
            " is used.\n\nThe list of available zones as well as the default value can"
            " be found by using the [List"
            " zones](https://docs.databricks.com/dev-tools/api/latest/clusters.html#list-zones)"
            " API."
        ),
    )

CanManage

Bases: str, Enum

Permission to manage the job.

Source code in prefect_databricks/models/jobs.py

class CanManage(str, Enum):
    """
    Permission to manage the job.
    """

    canmanage = "CAN_MANAGE"

CanManageRun

Bases: str, Enum

Permission to run and/or manage runs for the job.

Source code in prefect_databricks/models/jobs.py

class CanManageRun(str, Enum):
    """
    Permission to run and/or manage runs for the job.
    """

    canmanagerun = "CAN_MANAGE_RUN"

CanView

Bases: str, Enum

Permission to view the settings of the job.

Source code in prefect_databricks/models/jobs.py

class CanView(str, Enum):
    """
    Permission to view the settings of the job.
    """

    canview = "CAN_VIEW"

ClusterAttributes

Bases: BaseModel

See source code for the fields' description.

Source code in prefect_databricks/models/jobs.py

class ClusterAttributes(BaseModel):
    """
    See source code for the fields' description.
    """

    class Config:
        extra = Extra.allow
        allow_mutation = False

    autotermination_minutes: Optional[int] = Field(
        None,
        description=(
            "Automatically terminates the cluster after it is inactive for this time in"
            " minutes. If not set, this cluster is not be automatically terminated. If"
            " specified, the threshold must be between 10 and 10000 minutes. You can"
            " also set this value to 0 to explicitly disable automatic termination."
        ),
    )
    aws_attributes: Optional[AwsAttributes] = Field(
        None,
        description=(
            "Attributes related to clusters running on Amazon Web Services. If not"
            " specified at cluster creation, a set of default values are used."
        ),
    )
    cluster_log_conf: Optional[ClusterLogConf] = Field(
        None,
        description=(
            "The configuration for delivering Spark logs to a long-term storage"
            " destination. Only one destination can be specified for one cluster. If"
            " the conf is given, the logs is delivered to the destination every `5"
            " mins`. The destination of driver logs is"
            " `<destination>/<cluster-ID>/driver`, while the destination of executor"
            " logs is `<destination>/<cluster-ID>/executor`."
        ),
    )
    cluster_name: Optional[str] = Field(
        None,
        description=(
            "Cluster name requested by the user. This doesn’t have to be unique. If not"
            " specified at creation, the cluster name is an empty string."
        ),
    )
    cluster_source: Optional[ClusterSource] = Field(
        None,
        description=(
            "Determines whether the cluster was created by a user through the UI,"
            " created by the Databricks Jobs scheduler, or through an API request."
        ),
    )
    custom_tags: Optional[ClusterTag] = Field(
        None,
        description=(
            "An object containing a set of tags for cluster resources. Databricks tags"
            " all cluster resources (such as AWS instances and EBS volumes) with these"
            " tags in addition to default_tags.\n\n**Note**:\n\n* Tags are not"
            " supported on legacy node types such as compute-optimized and"
            " memory-optimized\n* Databricks allows at most 45 custom tags"
        ),
    )
    docker_image: Optional[DockerImage] = Field(
        None,
        description=(
            "Docker image for a [custom"
            " container](https://docs.databricks.com/clusters/custom-containers.html)."
        ),
    )
    driver_node_type_id: Optional[str] = Field(
        None,
        description=(
            "The node type of the Spark driver. This field is optional; if unset, the"
            " driver node type is set as the same value as `node_type_id` defined"
            " above."
        ),
    )
    enable_elastic_disk: Optional[bool] = Field(
        None,
        description=(
            "Autoscaling Local Storage: when enabled, this cluster dynamically acquires"
            " additional disk space when its Spark workers are running low on disk"
            " space. This feature requires specific AWS permissions to function"
            " correctly. Refer to [Autoscaling local"
            " storage](https://docs.databricks.com/clusters/configure.html#autoscaling-local-storage)"
            " for details."
        ),
    )
    enable_local_disk_encryption: Optional[bool] = Field(
        None,
        description=(
            "Determines whether encryption of the disks attached to the cluster locally"
            " is enabled."
        ),
    )
    init_scripts: Optional[List[InitScriptInfo]] = Field(
        None,
        description=(
            "The configuration for storing init scripts. Any number of destinations can"
            " be specified. The scripts are executed sequentially in the order"
            " provided. If `cluster_log_conf` is specified, init script logs are sent"
            " to `<destination>/<cluster-ID>/init_scripts`."
        ),
    )
    instance_pool_id: Optional[str] = Field(
        None,
        description=(
            "The optional ID of the instance pool to which the cluster belongs. Refer"
            " to [Pools](https://docs.databricks.com/clusters/instance-pools/index.html)"
            " for details."
        ),
    )
    node_type_id: Optional[str] = Field(
        None,
        description=(
            "This field encodes, through a single value, the resources available to"
            " each of the Spark nodes in this cluster. For example, the Spark nodes can"
            " be provisioned and optimized for memory or compute intensive workloads A"
            " list of available node types can be retrieved by using the [List node"
            " types](https://docs.databricks.com/dev-tools/api/latest/clusters.html#list-node-types)"
            " API call."
        ),
    )
    policy_id: Optional[str] = Field(
        None,
        description=(
            "A [cluster"
            " policy](https://docs.databricks.com/dev-tools/api/latest/policies.html) ID."
        ),
    )
    spark_conf: Optional[SparkConfPair] = Field(
        None,
        description=(
            "An object containing a set of optional, user-specified Spark configuration"
            " key-value pairs. You can also pass in a string of extra JVM options to"
            " the driver and the executors via `spark.driver.extraJavaOptions` and"
            " `spark.executor.extraJavaOptions` respectively.\n\nExample Spark confs:"
            ' `{"spark.speculation": true, "spark.streaming.ui.retainedBatches": 5}` or'
            ' `{"spark.driver.extraJavaOptions": "-verbose:gc -XX:+PrintGCDetails"}`'
        ),
    )
    spark_env_vars: Optional[SparkEnvPair] = Field(
        None,
        description=(
            "An object containing a set of optional, user-specified environment"
            " variable key-value pairs. Key-value pairs of the form (X,Y) are exported"
            " as is (that is, `export X='Y'`) while launching the driver and"
            " workers.\n\nIn order to specify an additional set of"
            " `SPARK_DAEMON_JAVA_OPTS`, we recommend appending them to"
            " `$SPARK_DAEMON_JAVA_OPTS` as shown in the following example. This ensures"
            " that all default databricks managed environmental variables are included"
            ' as well.\n\nExample Spark environment variables: `{"SPARK_WORKER_MEMORY":'
            ' "28000m", "SPARK_LOCAL_DIRS": "/local_disk0"}` or'
            ' `{"SPARK_DAEMON_JAVA_OPTS": "$SPARK_DAEMON_JAVA_OPTS'
            ' -Dspark.shuffle.service.enabled=true"}`'
        ),
    )
    spark_version: Optional[str] = Field(
        None,
        description=(
            "The runtime version of the cluster, for example “5.0.x-scala2.11”. You can"
            " retrieve a list of available runtime versions by using the [Runtime"
            " versions](https://docs.databricks.com/dev-tools/api/latest/clusters.html#runtime-versions)"
            " API call."
        ),
    )
    ssh_public_keys: Optional[List[str]] = Field(
        None,
        description=(
            "SSH public key contents that is added to each Spark node in this cluster."
            " The corresponding private keys can be used to login with the user name"
            " `ubuntu` on port `2200`. Up to 10 keys can be specified."
        ),
    )

ClusterCloudProviderNodeInfo

Bases: BaseModel

See source code for the fields' description.

Source code in prefect_databricks/models/jobs.py

class ClusterCloudProviderNodeInfo(BaseModel):
    """
    See source code for the fields' description.
    """

    class Config:
        extra = Extra.allow
        allow_mutation = False

    available_core_quota: Optional[int] = Field(
        None, description="Available CPU core quota."
    )
    status: Optional[ClusterCloudProviderNodeStatus] = Field(
        None, description="Status as reported by the cloud provider."
    )
    total_core_quota: Optional[int] = Field(None, description="Total CPU core quota.")

ClusterCloudProviderNodeStatus

Bases: str, Enum

* NotEnabledOnSubscription: Node type not available for subscription.

• NotAvailableInRegion: Node type not available in region.

Source code in prefect_databricks/models/jobs.py

class ClusterCloudProviderNodeStatus(str, Enum):
    """
        * NotEnabledOnSubscription: Node type not available for subscription.
    * NotAvailableInRegion: Node type not available in region.

    """

    not_enabled_on_subscription = "NotEnabledOnSubscription"
    not_available_in_region = "NotAvailableInRegion"

ClusterEvent

Bases: BaseModel

See source code for the fields' description.

Source code in prefect_databricks/models/jobs.py

class ClusterEvent(BaseModel):
    """
    See source code for the fields' description.
    """

    class Config:
        extra = Extra.allow
        allow_mutation = False

    cluster_id: str = Field(
        ..., description="Canonical identifier for the cluster. This field is required."
    )
    details: EventDetails = Field(
        ..., description="The event details. This field is required."
    )
    timestamp: Optional[int] = Field(
        None,
        description=(
            "The timestamp when the event occurred, stored as the number of"
            " milliseconds since the unix epoch. Assigned by the Timeline service."
        ),
    )
    type: ClusterEventType = Field(
        ..., description="The event type. This field is required."
    )

ClusterEventType

Bases: str, Enum

* `CREATING`: Indicates that the cluster is being created.

• DID_NOT_EXPAND_DISK: Indicates that a disk is low on space, but adding disks would put it over the max capacity.
• EXPANDED_DISK: Indicates that a disk was low on space and the disks were expanded.
• FAILED_TO_EXPAND_DISK: Indicates that a disk was low on space and disk space could not be expanded.
• INIT_SCRIPTS_STARTING: Indicates that the cluster scoped init script has started.
• INIT_SCRIPTS_FINISHED: Indicates that the cluster scoped init script has finished.
• STARTING: Indicates that the cluster is being started.
• RESTARTING: Indicates that the cluster is being started.
• TERMINATING: Indicates that the cluster is being terminated.
• EDITED: Indicates that the cluster has been edited.
• RUNNING: Indicates the cluster has finished being created. Includes the number of nodes in the cluster and a failure reason if some nodes could not be acquired.
• RESIZING: Indicates a change in the target size of the cluster (upsize or downsize).
• UPSIZE_COMPLETED: Indicates that nodes finished being added to the cluster. Includes the number of nodes in the cluster and a failure reason if some nodes could not be acquired.
• NODES_LOST: Indicates that some nodes were lost from the cluster.
• DRIVER_HEALTHY: Indicates that the driver is healthy and the cluster is ready for use.
• DRIVER_UNAVAILABLE: Indicates that the driver is unavailable.
• SPARK_EXCEPTION: Indicates that a Spark exception was thrown from the driver.
• DRIVER_NOT_RESPONDING: Indicates that the driver is up but is not responsive, likely due to GC.
• DBFS_DOWN: Indicates that the driver is up but DBFS is down.
• METASTORE_DOWN: Indicates that the driver is up but the metastore is down.
• NODE_BLACKLISTED: Indicates that a node is not allowed by Spark.
• PINNED: Indicates that the cluster was pinned.
• UNPINNED: Indicates that the cluster was unpinned.

Source code in prefect_databricks/models/jobs.py

class ClusterEventType(str, Enum):
    """
        * `CREATING`: Indicates that the cluster is being created.
    * `DID_NOT_EXPAND_DISK`: Indicates that a disk is low on space, but adding disks would put it over the max capacity.
    * `EXPANDED_DISK`: Indicates that a disk was low on space and the disks were expanded.
    * `FAILED_TO_EXPAND_DISK`: Indicates that a disk was low on space and disk space could not be expanded.
    * `INIT_SCRIPTS_STARTING`: Indicates that the cluster scoped init script has started.
    * `INIT_SCRIPTS_FINISHED`: Indicates that the cluster scoped init script has finished.
    * `STARTING`: Indicates that the cluster is being started.
    * `RESTARTING`: Indicates that the cluster is being started.
    * `TERMINATING`: Indicates that the cluster is being terminated.
    * `EDITED`: Indicates that the cluster has been edited.
    * `RUNNING`: Indicates the cluster has finished being created. Includes the number of nodes in the cluster and a failure reason if some nodes could not be acquired.
    * `RESIZING`: Indicates a change in the target size of the cluster (upsize or downsize).
    * `UPSIZE_COMPLETED`: Indicates that nodes finished being added to the cluster. Includes the number of nodes in the cluster and a failure reason if some nodes could not be acquired.
    * `NODES_LOST`: Indicates that some nodes were lost from the cluster.
    * `DRIVER_HEALTHY`: Indicates that the driver is healthy and the cluster is ready for use.
    * `DRIVER_UNAVAILABLE`: Indicates that the driver is unavailable.
    * `SPARK_EXCEPTION`: Indicates that a Spark exception was thrown from the driver.
    * `DRIVER_NOT_RESPONDING`: Indicates that the driver is up but is not responsive, likely due to GC.
    * `DBFS_DOWN`: Indicates that the driver is up but DBFS is down.
    * `METASTORE_DOWN`: Indicates that the driver is up but the metastore is down.
    * `NODE_BLACKLISTED`: Indicates that a node is not allowed by Spark.
    * `PINNED`: Indicates that the cluster was pinned.
    * `UNPINNED`: Indicates that the cluster was unpinned.
    """

    creating = "CREATING"
    didnotexpanddisk = "DID_NOT_EXPAND_DISK"
    expandeddisk = "EXPANDED_DISK"
    failedtoexpanddisk = "FAILED_TO_EXPAND_DISK"
    initscriptsstarting = "INIT_SCRIPTS_STARTING"
    initscriptsfinished = "INIT_SCRIPTS_FINISHED"
    starting = "STARTING"
    restarting = "RESTARTING"
    terminating = "TERMINATING"
    edited = "EDITED"
    running = "RUNNING"
    resizing = "RESIZING"
    upsizecompleted = "UPSIZE_COMPLETED"
    nodeslost = "NODES_LOST"
    driverhealthy = "DRIVER_HEALTHY"
    driverunavailable = "DRIVER_UNAVAILABLE"
    sparkexception = "SPARK_EXCEPTION"
    drivernotresponding = "DRIVER_NOT_RESPONDING"
    dbfsdown = "DBFS_DOWN"
    metastoredown = "METASTORE_DOWN"
    nodeblacklisted = "NODE_BLACKLISTED"
    pinned = "PINNED"
    unpinned = "UNPINNED"

ClusterInfo

Bases: BaseModel

See source code for the fields' description.

Source code in prefect_databricks/models/jobs.py

class ClusterInfo(BaseModel):
    """
    See source code for the fields' description.
    """

    class Config:
        extra = Extra.allow
        allow_mutation = False

    autoscale: Optional[AutoScale] = Field(
        None,
        description=(
            "If autoscale, parameters needed in order to automatically scale clusters"
            " up and down based on load."
        ),
    )
    autotermination_minutes: Optional[int] = Field(
        None,
        description=(
            "Automatically terminates the cluster after it is inactive for this time in"
            " minutes. If not set, this cluster is not be automatically terminated. If"
            " specified, the threshold must be between 10 and 10000 minutes. You can"
            " also set this value to 0 to explicitly disable automatic termination."
        ),
    )
    aws_attributes: Optional[AwsAttributes] = Field(
        None,
        description=(
            "Attributes related to clusters running on Amazon Web Services. If not"
            " specified at cluster creation, a set of default values is used."
        ),
    )
    cluster_cores: Optional[float] = Field(
        None,
        description=(
            "Number of CPU cores available for this cluster. This can be fractional"
            " since certain node types are configured to share cores between Spark"
            " nodes on the same instance."
        ),
    )
    cluster_id: Optional[str] = Field(
        None,
        description=(
            "Canonical identifier for the cluster. This ID is retained during cluster"
            " restarts and resizes, while each new cluster has a globally unique ID."
        ),
    )
    cluster_log_conf: Optional[ClusterLogConf] = Field(
        None,
        description=(
            "The configuration for delivering Spark logs to a long-term storage"
            " destination. Only one destination can be specified for one cluster. If"
            " the conf is given, the logs are delivered to the destination every `5"
            " mins`. The destination of driver logs is"
            " `<destination>/<cluster-ID>/driver`, while the destination of executor"
            " logs is `<destination>/<cluster-ID>/executor`."
        ),
    )
    cluster_log_status: Optional[LogSyncStatus] = Field(
        None, description="Cluster log delivery status."
    )
    cluster_memory_mb: Optional[int] = Field(
        None, description="Total amount of cluster memory, in megabytes."
    )
    cluster_name: Optional[str] = Field(
        None,
        description=(
            "Cluster name requested by the user. This doesn’t have to be unique. If not"
            " specified at creation, the cluster name is an empty string."
        ),
    )
    cluster_source: Optional[ClusterSource] = Field(
        None,
        description=(
            "Determines whether the cluster was created by a user through the UI, by"
            " the Databricks Jobs scheduler, or through an API request."
        ),
    )
    creator_user_name: Optional[str] = Field(
        None,
        description=(
            "Creator user name. The field won’t be included in the response if the user"
            " has already been deleted."
        ),
    )
    custom_tags: Optional[List[ClusterTag]] = Field(
        None,
        description=(
            "An object containing a set of tags for cluster resources. Databricks tags"
            " all cluster resources (such as AWS instances and EBS volumes) with these"
            " tags in addition to default_tags.\n\n**Note**:\n\n* Tags are not"
            " supported on legacy node types such as compute-optimized and"
            " memory-optimized\n* Databricks allows at most 45 custom tags"
        ),
    )
    default_tags: Optional[ClusterTag] = Field(
        None,
        description=(
            "An object containing a set of tags that are added by Databricks regardless"
            " of any custom_tags, including:\n\n* Vendor: Databricks\n* Creator:"
            " <username-of-creator>\n* ClusterName: <name-of-cluster>\n* ClusterId:"
            " <id-of-cluster>\n* Name: <Databricks internal use>  \nOn job clusters:\n*"
            " RunName: <name-of-job>\n* JobId: <id-of-job>  \nOn resources used by"
            " Databricks SQL:\n* SqlEndpointId: <id-of-endpoint>"
        ),
    )
    docker_image: Optional[DockerImage] = Field(
        None,
        description=(
            "Docker image for a [custom"
            " container](https://docs.databricks.com/clusters/custom-containers.html)."
        ),
    )
    driver: Optional[SparkNode] = Field(
        None,
        description=(
            "Node on which the Spark driver resides. The driver node contains the Spark"
            " master and the Databricks application that manages the per-notebook Spark"
            " REPLs."
        ),
    )
    driver_node_type_id: Optional[str] = Field(
        None,
        description=(
            "The node type of the Spark driver. This field is optional; if unset, the"
            " driver node type is set as the same value as `node_type_id` defined"
            " above."
        ),
    )
    enable_elastic_disk: Optional[bool] = Field(
        None,
        description=(
            "Autoscaling Local Storage: when enabled, this cluster dynamically acquires"
            " additional disk space when its Spark workers are running low on disk"
            " space. This feature requires specific AWS permissions to function"
            " correctly - refer to [Autoscaling local"
            " storage](https://docs.databricks.com/clusters/configure.html#autoscaling-local-storage)"
            " for details."
        ),
    )
    executors: Optional[List[SparkNode]] = Field(
        None, description="Nodes on which the Spark executors reside."
    )
    init_scripts: Optional[List[InitScriptInfo]] = Field(
        None,
        description=(
            "The configuration for storing init scripts. Any number of destinations can"
            " be specified. The scripts are executed sequentially in the order"
            " provided. If `cluster_log_conf` is specified, init script logs are sent"
            " to `<destination>/<cluster-ID>/init_scripts`."
        ),
    )
    instance_pool_id: Optional[str] = Field(
        None,
        description=(
            "The optional ID of the instance pool to which the cluster belongs. Refer"
            " to [Pools](https://docs.databricks.com/clusters/instance-pools/index.html)"
            " for details."
        ),
    )
    jdbc_port: Optional[int] = Field(
        None,
        description=(
            "Port on which Spark JDBC server is listening in the driver node. No"
            " service listens on this port in executor nodes."
        ),
    )
    last_activity_time: Optional[int] = Field(
        None,
        description=(
            "Time (in epoch milliseconds) when the cluster was last active. A cluster"
            " is active if there is at least one command that has not finished on the"
            " cluster. This field is available after the cluster has reached a"
            " `RUNNING` state. Updates to this field are made as best-effort attempts."
            " Certain versions of Spark do not support reporting of cluster activity."
            " Refer to [Automatic"
            " termination](https://docs.databricks.com/clusters/clusters-manage.html#automatic-termination)"
            " for details."
        ),
    )
    last_state_loss_time: Optional[int] = Field(
        None,
        description=(
            "Time when the cluster driver last lost its state (due to a restart or"
            " driver failure)."
        ),
    )
    node_type_id: Optional[str] = Field(
        None,
        description=(
            "This field encodes, through a single value, the resources available to"
            " each of the Spark nodes in this cluster. For example, the Spark nodes can"
            " be provisioned and optimized for memory or compute intensive workloads. A"
            " list of available node types can be retrieved by using the [List node"
            " types](https://docs.databricks.com/dev-tools/api/latest/clusters.html#list-node-types)"
            " API call."
        ),
    )
    num_workers: Optional[int] = Field(
        None,
        description=(
            "If num_workers, number of worker nodes that this cluster must have. A"
            " cluster has one Spark driver and num_workers executors for a total of"
            " num_workers + 1 Spark nodes. **Note:** When reading the properties of a"
            " cluster, this field reflects the desired number of workers rather than"
            " the actual number of workers. For instance, if a cluster is resized from"
            " 5 to 10 workers, this field is immediately updated to reflect the target"
            " size of 10 workers, whereas the workers listed in `executors` gradually"
            " increase from 5 to 10 as the new nodes are provisioned."
        ),
    )
    spark_conf: Optional[SparkConfPair] = Field(
        None,
        description=(
            "An object containing a set of optional, user-specified Spark configuration"
            " key-value pairs. You can also pass in a string of extra JVM options to"
            " the driver and the executors via `spark.driver.extraJavaOptions` and"
            " `spark.executor.extraJavaOptions` respectively.\n\nExample Spark confs:"
            ' `{"spark.speculation": true, "spark.streaming.ui.retainedBatches": 5}` or'
            ' `{"spark.driver.extraJavaOptions": "-verbose:gc -XX:+PrintGCDetails"}`'
        ),
    )
    spark_context_id: Optional[int] = Field(
        None,
        description=(
            "A canonical SparkContext identifier. This value _does_ change when the"
            " Spark driver restarts. The pair `(cluster_id, spark_context_id)` is a"
            " globally unique identifier over all Spark contexts."
        ),
    )
    spark_env_vars: Optional[SparkEnvPair] = Field(
        None,
        description=(
            "An object containing a set of optional, user-specified environment"
            " variable key-value pairs. Key-value pairs of the form (X,Y) are exported"
            " as is (that is, `export X='Y'`) while launching the driver and"
            " workers.\n\nTo specify an additional set of `SPARK_DAEMON_JAVA_OPTS`, we"
            " recommend appending them to `$SPARK_DAEMON_JAVA_OPTS` as shown in the"
            " following example. This ensures that all default databricks managed"
            " environmental variables are included as well.\n\nExample Spark"
            ' environment variables: `{"SPARK_WORKER_MEMORY": "28000m",'
            ' "SPARK_LOCAL_DIRS": "/local_disk0"}` or `{"SPARK_DAEMON_JAVA_OPTS":'
            ' "$SPARK_DAEMON_JAVA_OPTS -Dspark.shuffle.service.enabled=true"}`'
        ),
    )
    spark_version: Optional[str] = Field(
        None,
        description=(
            "The runtime version of the cluster. You can retrieve a list of available"
            " runtime versions by using the [Runtime"
            " versions](https://docs.databricks.com/dev-tools/api/latest/clusters.html#runtime-versions)"
            " API call."
        ),
    )
    ssh_public_keys: Optional[List[str]] = Field(
        None,
        description=(
            "SSH public key contents that are added to each Spark node in this cluster."
            " The corresponding private keys can be used to login with the user name"
            " `ubuntu` on port `2200`. Up to 10 keys can be specified."
        ),
    )
    start_time: Optional[int] = Field(
        None,
        description=(
            "Time (in epoch milliseconds) when the cluster creation request was"
            " received (when the cluster entered a `PENDING` state)."
        ),
    )
    state: Optional[ClusterState] = Field(None, description="State of the cluster.")
    state_message: Optional[str] = Field(
        None,
        description=(
            "A message associated with the most recent state transition (for example,"
            " the reason why the cluster entered a `TERMINATED` state). This field is"
            " unstructured, and its exact format is subject to change."
        ),
    )
    terminated_time: Optional[int] = Field(
        None,
        description=(
            "Time (in epoch milliseconds) when the cluster was terminated, if"
            " applicable."
        ),
    )
    termination_reason: Optional[TerminationReason] = Field(
        None,
        description=(
            "Information about why the cluster was terminated. This field only appears"
            " when the cluster is in a `TERMINATING` or `TERMINATED` state."
        ),
    )

ClusterInstance

Bases: BaseModel

See source code for the fields' description.

Source code in prefect_databricks/models/jobs.py

class ClusterInstance(BaseModel):
    """
    See source code for the fields' description.
    """

    class Config:
        extra = Extra.allow
        allow_mutation = False

    cluster_id: Optional[str] = Field(
        None,
        description=(
            "The canonical identifier for the cluster used by a run. This field is"
            " always available for runs on existing clusters. For runs on new clusters,"
            " it becomes available once the cluster is created. This value can be used"
            " to view logs by browsing to `/#setting/sparkui/$cluster_id/driver-logs`."
            " The logs continue to be available after the run completes.\n\nThe"
            " response won’t include this field if the identifier is not available yet."
        ),
        example="0923-164208-meows279",
    )
    spark_context_id: Optional[str] = Field(
        None,
        description=(
            "The canonical identifier for the Spark context used by a run. This field"
            " is filled in once the run begins execution. This value can be used to"
            " view the Spark UI by browsing to"
            " `/#setting/sparkui/$cluster_id/$spark_context_id`. The Spark UI continues"
            " to be available after the run has completed.\n\nThe response won’t"
            " include this field if the identifier is not available yet."
        ),
    )

ClusterLibraryStatuses

Bases: BaseModel

See source code for the fields' description.

Source code in prefect_databricks/models/jobs.py

class ClusterLibraryStatuses(BaseModel):
    """
    See source code for the fields' description.
    """

    class Config:
        extra = Extra.allow
        allow_mutation = False

    cluster_id: Optional[str] = Field(
        None, description="Unique identifier for the cluster."
    )
    library_statuses: Optional[List[LibraryFullStatus]] = Field(
        None, description="Status of all libraries on the cluster."
    )

ClusterLogConf

Bases: BaseModel

See source code for the fields' description.

Source code in prefect_databricks/models/jobs.py

class ClusterLogConf(BaseModel):
    """
    See source code for the fields' description.
    """

    class Config:
        extra = Extra.allow
        allow_mutation = False

    dbfs: Optional[DbfsStorageInfo] = Field(
        None,
        description=(
            "DBFS location of cluster log. Destination must be provided. For example,"
            ' `{ "dbfs" : { "destination" : "dbfs:/home/cluster_log" } }`'
        ),
    )
    s3: Optional[S3StorageInfo] = Field(
        None,
        description=(
            "S3 location of cluster log. `destination` and either `region` or"
            ' `endpoint` must be provided. For example, `{ "s3": { "destination" :'
            ' "s3://cluster_log_bucket/prefix", "region" : "us-west-2" } }`'
        ),
    )

ClusterSize

Bases: BaseModel

See source code for the fields' description.

Source code in prefect_databricks/models/jobs.py

class ClusterSize(BaseModel):
    """
    See source code for the fields' description.
    """

    class Config:
        extra = Extra.allow
        allow_mutation = False

    autoscale: Optional[AutoScale] = Field(
        None,
        description=(
            "If autoscale, parameters needed in order to automatically scale clusters"
            " up and down based on load."
        ),
    )
    num_workers: Optional[int] = Field(
        None,
        description=(
            "If num_workers, number of worker nodes that this cluster must have. A"
            " cluster has one Spark driver and num_workers executors for a total of"
            " num_workers + 1 Spark nodes. When reading the properties of a cluster,"
            " this field reflects the desired number of workers rather than the actual"
            " number of workers. For instance, if a cluster is resized from 5 to 10"
            " workers, this field is updated to reflect the target size of 10 workers,"
            " whereas the workers listed in executors gradually increase from 5 to 10"
            " as the new nodes are provisioned."
        ),
    )

ClusterSource

Bases: str, Enum

* UI: Cluster created through the UI.

• JOB: Cluster created by the Databricks job scheduler.
• API: Cluster created through an API call.

Source code in prefect_databricks/models/jobs.py

class ClusterSource(str, Enum):
    """
        * UI: Cluster created through the UI.
    * JOB: Cluster created by the Databricks job scheduler.
    * API: Cluster created through an API call.

    """

    ui = "UI"
    job = "JOB"
    api = "API"

ClusterSpec

Bases: BaseModel

See source code for the fields' description.

Source code in prefect_databricks/models/jobs.py

class ClusterSpec(BaseModel):
    """
    See source code for the fields' description.
    """

    class Config:
        extra = Extra.allow
        allow_mutation = False

    existing_cluster_id: Optional[str] = Field(
        None,
        description=(
            "If existing_cluster_id, the ID of an existing cluster that is used for all"
            " runs of this job. When running jobs on an existing cluster, you may need"
            " to manually restart the cluster if it stops responding. We suggest"
            " running jobs on new clusters for greater reliability."
        ),
        example="0923-164208-meows279",
    )
    libraries: Optional[List[Library]] = Field(
        None,
        description=(
            "An optional list of libraries to be installed on the cluster that executes"
            " the job. The default value is an empty list."
        ),
    )
    new_cluster: Optional[NewCluster] = Field(
        None,
        description=(
            "If new_cluster, a description of a cluster that is created for each run."
        ),
    )

ClusterState

Bases: str, Enum

* PENDING: Indicates that a cluster is in the process of being created.

• RUNNING: Indicates that a cluster has been started and is ready for use.
• RESTARTING: Indicates that a cluster is in the process of restarting.
• RESIZING: Indicates that a cluster is in the process of adding or removing nodes.
• TERMINATING: Indicates that a cluster is in the process of being destroyed.
• TERMINATED: Indicates that a cluster has been successfully destroyed.
• ERROR: This state is no longer used. It was used to indicate a cluster that failed to be created. TERMINATING and TERMINATED are used instead.
• UNKNOWN: Indicates that a cluster is in an unknown state. A cluster should never be in this state.

Source code in prefect_databricks/models/jobs.py

class ClusterState(str, Enum):
    """
        * PENDING: Indicates that a cluster is in the process of being created.
    * RUNNING: Indicates that a cluster has been started and is ready for use.
    * RESTARTING: Indicates that a cluster is in the process of restarting.
    * RESIZING: Indicates that a cluster is in the process of adding or removing nodes.
    * TERMINATING: Indicates that a cluster is in the process of being destroyed.
    * TERMINATED: Indicates that a cluster has been successfully destroyed.
    * ERROR: This state is no longer used. It was used to indicate a cluster that failed to be created. `TERMINATING` and `TERMINATED` are used instead.
    * UNKNOWN: Indicates that a cluster is in an unknown state. A cluster should never be in this state.

    """

    pending = "PENDING"
    running = "RUNNING"
    restarting = "RESTARTING"
    resizing = "RESIZING"
    terminating = "TERMINATING"
    terminated = "TERMINATED"
    error = "ERROR"
    unknown = "UNKNOWN"

ClusterTag

Bases: BaseModel

See source code for the fields' description.

An object with key value pairs. The key length must be between 1 and 127 UTF-8 characters, inclusive. The value length must be less than or equal to 255 UTF-8 characters. For a list of all restrictions, see AWS Tag Restrictions: https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/Using_Tags.html#tag-restrictions

Source code in prefect_databricks/models/jobs.py

class ClusterTag(BaseModel):
    """
    See source code for the fields' description.

    An object with key value pairs. The key length must be between 1 and 127 UTF-8 characters, inclusive. The value length must be less than or equal to 255 UTF-8 characters. For a list of all restrictions, see AWS Tag Restrictions: <https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/Using_Tags.html#tag-restrictions>
    """

    class Config:
        extra = Extra.allow

        allow_mutation = False

CronSchedule

Bases: BaseModel

See source code for the fields' description.

Source code in prefect_databricks/models/jobs.py

class CronSchedule(BaseModel):
    """
    See source code for the fields' description.
    """

    class Config:
        extra = Extra.allow
        allow_mutation = False

    pause_status: Optional[Literal["PAUSED", "UNPAUSED"]] = Field(
        None,
        description="Indicate whether this schedule is paused or not.",
        example="PAUSED",
    )
    quartz_cron_expression: str = Field(
        ...,
        description=(
            "A Cron expression using Quartz syntax that describes the schedule for a"
            " job. See [Cron"
            " Trigger](http://www.quartz-scheduler.org/documentation/quartz-2.3.0/tutorials/crontrigger.html)"
            " for details. This field is required."
        ),
        example="20 30 * * * ?",
    )
    timezone_id: str = Field(
        ...,
        description=(
            "A Java timezone ID. The schedule for a job is resolved with respect to"
            " this timezone. See [Java"
            " TimeZone](https://docs.oracle.com/javase/7/docs/api/java/util/TimeZone.html)"
            " for details. This field is required."
        ),
        example="Europe/London",
    )

DbfsStorageInfo

Bases: BaseModel

See source code for the fields' description.

Source code in prefect_databricks/models/jobs.py

class DbfsStorageInfo(BaseModel):
    """
    See source code for the fields' description.
    """

    class Config:
        extra = Extra.allow
        allow_mutation = False

    destination: Optional[str] = Field(
        None, description="DBFS destination. Example: `dbfs:/my/path`"
    )

DbtOutput

Bases: BaseModel

See source code for the fields' description.

Source code in prefect_databricks/models/jobs.py

class DbtOutput(BaseModel):
    """
    See source code for the fields' description.
    """

    class Config:
        extra = Extra.allow
        allow_mutation = False

    artifacts_headers: Optional[Dict[str, Any]] = Field(
        None,
        description=(
            "An optional map of headers to send when retrieving the artifact from the"
            " `artifacts_link`."
        ),
    )
    artifacts_link: Optional[str] = Field(
        None,
        description=(
            "A pre-signed URL to download the (compressed) dbt artifacts. This link is"
            " valid for a limited time (30 minutes). This information is only available"
            " after the run has finished."
        ),
    )

DbtTask

Bases: BaseModel

See source code for the fields' description.

Source code in prefect_databricks/models/jobs.py

class DbtTask(BaseModel):
    """
    See source code for the fields' description.
    """

    class Config:
        extra = Extra.allow
        allow_mutation = False

    catalog: Optional[str] = Field(
        None,
        description=(
            "Optional name of the catalog to use. The value is the top level in the"
            " 3-level namespace of Unity Catalog (catalog / schema / relation). The"
            " catalog value can only be specified if a warehouse_id is specified."
            " Requires dbt-databricks >= 1.1.1."
        ),
        example="main",
    )
    commands: List = Field(
        ...,
        description=(
            "A list of dbt commands to execute. All commands must start with `dbt`."
            " This parameter must not be empty. A maximum of up to 10 commands can be"
            " provided."
        ),
        example=["dbt deps", "dbt seed", "dbt run --models 123"],
    )
    profiles_directory: Optional[str] = Field(
        None,
        description=(
            "Optional (relative) path to the profiles directory. Can only be specified"
            " if no warehouse_id is specified. If no warehouse_id is specified and this"
            " folder is unset, the root directory is used."
        ),
    )
    project_directory: Optional[str] = Field(
        None,
        description=(
            "Optional (relative) path to the project directory, if no value is"
            " provided, the root of the git repository is used."
        ),
    )
    schema_: Optional[str] = Field(
        None,
        alias="schema",
        description=(
            "Optional schema to write to. This parameter is only used when a"
            " warehouse_id is also provided. If not provided, the `default` schema is"
            " used."
        ),
    )
    warehouse_id: Optional[str] = Field(
        None,
        description=(
            "ID of the SQL warehouse to connect to. If provided, we automatically"
            " generate and provide the profile and connection details to dbt. It can be"
            " overridden on a per-command basis by using the `--profiles-dir` command"
            " line argument."
        ),
        example="30dade0507d960d1",
    )

DockerBasicAuth

Bases: BaseModel

See source code for the fields' description.

Source code in prefect_databricks/models/jobs.py

class DockerBasicAuth(BaseModel):
    """
    See source code for the fields' description.
    """

    class Config:
        extra = Extra.allow
        allow_mutation = False

    password: Optional[str] = Field(
        None, description="Password for the Docker repository."
    )
    username: Optional[str] = Field(
        None, description="User name for the Docker repository."
    )

DockerImage

Bases: BaseModel

See source code for the fields' description.

Source code in prefect_databricks/models/jobs.py

class DockerImage(BaseModel):
    """
    See source code for the fields' description.
    """

    class Config:
        extra = Extra.allow
        allow_mutation = False

    basic_auth: Optional[DockerBasicAuth] = Field(
        None, description="Basic authentication information for Docker repository."
    )
    url: Optional[str] = Field(None, description="URL for the Docker image.")

Error

Bases: BaseModel

See source code for the fields' description.

Source code in prefect_databricks/models/jobs.py

class Error(BaseModel):
    """
    See source code for the fields' description.
    """

    class Config:
        extra = Extra.allow
        allow_mutation = False

    error_code: Optional[str] = Field(
        None, description="Error code", example="INTERNAL_ERROR"
    )
    message: Optional[str] = Field(
        None,
        description=(
            "Human-readable error message that describes the cause of the error."
        ),
        example="Unexpected error.",
    )

EventDetails

Bases: BaseModel

See source code for the fields' description.

Source code in prefect_databricks/models/jobs.py

class EventDetails(BaseModel):
    """
    See source code for the fields' description.
    """

    class Config:
        extra = Extra.allow
        allow_mutation = False

    attributes: Optional[AwsAttributes] = Field(
        None,
        description=(
            "* For created clusters, the attributes of the cluster.\n* For edited"
            " clusters, the new attributes of the cluster."
        ),
    )
    cause: Optional[ResizeCause] = Field(
        None, description="The cause of a change in target size."
    )
    cluster_size: Optional[ClusterSize] = Field(
        None,
        description="The cluster size that was set in the cluster creation or edit.",
    )
    current_num_workers: Optional[int] = Field(
        None, description="The number of nodes in the cluster."
    )
    previous_attributes: Optional[AwsAttributes] = Field(
        None, description="The cluster attributes before a cluster was edited."
    )
    previous_cluster_size: Optional[ClusterSize] = Field(
        None, description="The size of the cluster before an edit or resize."
    )
    reason: Optional[TerminationReason] = Field(
        None,
        description=(
            "A termination reason:\n\n* On a `TERMINATED` event, the reason for the"
            " termination.\n* On a `RESIZE_COMPLETE` event, indicates the reason that"
            " we failed to acquire some nodes."
        ),
    )
    target_num_workers: Optional[int] = Field(
        None, description="The targeted number of nodes in the cluster."
    )
    user: Optional[str] = Field(
        None,
        description=(
            "The user that caused the event to occur. (Empty if it was done by"
            " Databricks.)"
        ),
    )

FileStorageInfo

Bases: BaseModel

See source code for the fields' description.

Source code in prefect_databricks/models/jobs.py

class FileStorageInfo(BaseModel):
    """
    See source code for the fields' description.
    """

    class Config:
        extra = Extra.allow
        allow_mutation = False

    destination: Optional[str] = Field(
        None, description="File destination. Example: `file:/my/file.sh`"
    )

GitSnapshot

Bases: BaseModel

See source code for the fields' description.

Read-only state of the remote repository at the time the job was run. This field is only included on job runs.

Source code in prefect_databricks/models/jobs.py

class GitSnapshot(BaseModel):
    """
    See source code for the fields' description.

    Read-only state of the remote repository at the time the job was run. This field is only included on job runs.
    """

    class Config:
        extra = Extra.allow
        allow_mutation = False

    used_commit: Optional[str] = Field(
        None,
        description=(
            "Commit that was used to execute the run. If git_branch was specified, this"
            " points to the HEAD of the branch at the time of the run; if git_tag was"
            " specified, this points to the commit the tag points to."
        ),
        example="4506fdf41e9fa98090570a34df7a5bce163ff15f",
    )

GitSource

Bases: BaseModel

See source code for the fields' description.

This functionality is in Public Preview.

An optional specification for a remote repository containing the notebooks used by this job's notebook tasks.

Source code in prefect_databricks/models/jobs.py

class GitSource(BaseModel):
    """
    See source code for the fields' description.

        This functionality is in Public Preview.

    An optional specification for a remote repository containing the notebooks used by this job's notebook tasks.
    """

    class Config:
        extra = Extra.allow
        allow_mutation = False

    git_branch: Optional[str] = Field(
        None,
        description=(
            "Name of the branch to be checked out and used by this job. This field"
            " cannot be specified in conjunction with git_tag or git_commit.\nThe"
            " maximum length is 255 characters."
        ),
        example="main",
    )
    git_commit: Optional[str] = Field(
        None,
        description=(
            "Commit to be checked out and used by this job. This field cannot be"
            " specified in conjunction with git_branch or git_tag.\nThe maximum length"
            " is 64 characters."
        ),
        example="e0056d01",
    )
    git_provider: Optional[
        Literal[
            "gitHub",
            "bitbucketCloud",
            "azureDevOpsServices",
            "gitHubEnterprise",
            "bitbucketServer",
            "gitLab",
            "gitLabEnterpriseEdition",
            "awsCodeCommit",
        ]
    ] = Field(
        None,
        description=(
            "Unique identifier of the service used to host the Git repository. The"
            " value is case insensitive."
        ),
        example="github",
    )
    git_snapshot: Optional[GitSnapshot] = None
    git_tag: Optional[str] = Field(
        None,
        description=(
            "Name of the tag to be checked out and used by this job. This field cannot"
            " be specified in conjunction with git_branch or git_commit.\nThe maximum"
            " length is 255 characters."
        ),
        example="release-1.0.0",
    )
    git_url: Optional[str] = Field(
        None,
        description=(
            "URL of the repository to be cloned by this job.\nThe maximum length is 300"
            " characters."
        ),
        example="https://github.com/databricks/databricks-cli",
    )

GitSource1

Bases: BaseModel

See source code for the fields' description.

Source code in prefect_databricks/models/jobs.py

class GitSource1(BaseModel):
    """
    See source code for the fields' description.
    """

    class Config:
        extra = Extra.allow
        allow_mutation = False

    __root__: Union[GitSource, Any, Any, Any] = Field(
        ...,
        description=(
            "This functionality is in Public Preview.\n\nAn optional specification for"
            " a remote repository containing the notebooks used by this job's notebook"
            " tasks."
        ),
        example={
            "git_branch": "main",
            "git_provider": "gitHub",
            "git_url": "https://github.com/databricks/databricks-cli",
        },
    )

GroupName

Bases: BaseModel

See source code for the fields' description.

Source code in prefect_databricks/models/jobs.py

class GroupName(BaseModel):
    """
    See source code for the fields' description.
    """

    class Config:
        extra = Extra.allow
        allow_mutation = False

    __root__: str = Field(
        ...,
        description=(
            "Group name. There are two built-in groups: `users` for all users, and"
            " `admins` for administrators."
        ),
        example="users",
    )

InitScriptInfo

Bases: BaseModel

See source code for the fields' description.

Source code in prefect_databricks/models/jobs.py

class InitScriptInfo(BaseModel):
    """
    See source code for the fields' description.
    """

    class Config:
        extra = Extra.allow
        allow_mutation = False

    s3: Optional[S3StorageInfo] = Field(
        None,
        alias="S3",
        description=(
            "S3 location of init script. Destination and either region or endpoint must"
            ' be provided. For example, `{ "s3": { "destination" :'
            ' "s3://init_script_bucket/prefix", "region" : "us-west-2" } }`'
        ),
    )
    dbfs: Optional[DbfsStorageInfo] = Field(
        None,
        description=(
            "DBFS location of init script. Destination must be provided. For example,"
            ' `{ "dbfs" : { "destination" : "dbfs:/home/init_script" } }`'
        ),
    )
    file: Optional[FileStorageInfo] = Field(
        None,
        description=(
            "File location of init script. Destination must be provided. For example,"
            ' `{ "file" : { "destination" : "file:/my/local/file.sh" } }`'
        ),
    )

IsOwner

Bases: str, Enum

Perimssion that represents ownership of the job.

Source code in prefect_databricks/models/jobs.py

class IsOwner(str, Enum):
    """
    Perimssion that represents ownership of the job.
    """

    isowner = "IS_OWNER"

Job

Bases: BaseModel

See source code for the fields' description.

Source code in prefect_databricks/models/jobs.py

class Job(BaseModel):
    """
    See source code for the fields' description.
    """

    class Config:
        extra = Extra.allow
        allow_mutation = False

    created_time: Optional[int] = Field(
        None,
        description=(
            "The time at which this job was created in epoch milliseconds (milliseconds"
            " since 1/1/1970 UTC)."
        ),
        example=1601370337343,
    )
    creator_user_name: Optional[str] = Field(
        None,
        description=(
            "The creator user name. This field won’t be included in the response if the"
            " user has already been deleted."
        ),
        example="user.name@databricks.com",
    )
    job_id: Optional[int] = Field(
        None, description="The canonical identifier for this job.", example=11223344
    )
    settings: Optional[JobSettings] = Field(
        None,
        description=(
            "Settings for this job and all of its runs. These settings can be updated"
            " using the `resetJob` method."
        ),
    )

JobCluster

Bases: BaseModel

See source code for the fields' description.

Source code in prefect_databricks/models/jobs.py

class JobCluster(BaseModel):
    """
    See source code for the fields' description.
    """

    class Config:
        extra = Extra.allow
        allow_mutation = False

    job_cluster_key: str = Field(
        ...,
        description=(
            "A unique name for the job cluster. This field is required and must be"
            " unique within the job.\n`JobTaskSettings` may refer to this field to"
            " determine which cluster to launch for the task execution."
        ),
        example="auto_scaling_cluster",
        max_length=100,
        min_length=1,
        regex="^[\\w\\-]+$",
    )
    new_cluster: Optional[NewCluster] = None

JobEmailNotifications

Bases: BaseModel

See source code for the fields' description.

Source code in prefect_databricks/models/jobs.py

class JobEmailNotifications(BaseModel):
    """
    See source code for the fields' description.
    """

    class Config:
        extra = Extra.allow
        allow_mutation = False

    no_alert_for_skipped_runs: Optional[bool] = Field(
        None,
        description=(
            "If true, do not send email to recipients specified in `on_failure` if the"
            " run is skipped."
        ),
        example=False,
    )
    on_failure: Optional[List[str]] = Field(
        None,
        description=(
            "A list of email addresses to notify when a run completes unsuccessfully. A"
            " run is considered unsuccessful if it ends with an `INTERNAL_ERROR`"
            " `life_cycle_state` or a `SKIPPED`, `FAILED`, or `TIMED_OUT`"
            " `result_state`. If not specified on job creation, reset, or update, or"
            " the list is empty, then notifications are not sent. Job-level failure"
            " notifications are sent only once after the entire job run (including all"
            " of its retries) has failed. Notifications are not sent when failed job"
            " runs are retried. To receive a failure notification after every failed"
            " task (including every failed retry), use task-level notifications"
            " instead."
        ),
        example=["user.name@databricks.com"],
    )
    on_start: Optional[List[str]] = Field(
        None,
        description=(
            "A list of email addresses to be notified when a run begins. If not"
            " specified on job creation, reset, or update, the list is empty, and"
            " notifications are not sent."
        ),
        example=["user.name@databricks.com"],
    )
    on_success: Optional[List[str]] = Field(
        None,
        description=(
            "A list of email addresses to be notified when a run successfully"
            " completes. A run is considered to have completed successfully if it ends"
            " with a `TERMINATED` `life_cycle_state` and a `SUCCESSFUL` result_state."
            " If not specified on job creation, reset, or update, the list is empty,"
            " and notifications are not sent."
        ),
        example=["user.name@databricks.com"],
    )

JobSettings

Bases: BaseModel

See source code for the fields' description.

Source code in prefect_databricks/models/jobs.py

class JobSettings(BaseModel):
    """
    See source code for the fields' description.
    """

    class Config:
        extra = Extra.allow
        allow_mutation = False

    email_notifications: Optional[JobEmailNotifications] = Field(
        None,
        description=(
            "An optional set of email addresses that is notified when runs of this job"
            " begin or complete as well as when this job is deleted. The default"
            " behavior is to not send any emails."
        ),
    )
    format: Optional[Literal["SINGLE_TASK", "MULTI_TASK"]] = Field(
        None,
        description=(
            "Used to tell what is the format of the job. This field is ignored in"
            " Create/Update/Reset calls. When using the Jobs API 2.1 this value is"
            ' always set to `"MULTI_TASK"`.'
        ),
        example="MULTI_TASK",
    )
    git_source: Optional[GitSource1] = Field(
        None,
        description=(
            "This functionality is in Public Preview.\n\nAn optional specification for"
            " a remote repository containing the notebooks used by this job's notebook"
            " tasks."
        ),
        example={
            "git_branch": "main",
            "git_provider": "gitHub",
            "git_url": "https://github.com/databricks/databricks-cli",
        },
    )
    job_clusters: Optional[List[JobCluster]] = Field(
        None,
        description=(
            "A list of job cluster specifications that can be shared and reused by"
            " tasks of this job. Libraries cannot be declared in a shared job cluster."
            " You must declare dependent libraries in task settings."
        ),
        example=[
            {
                "job_cluster_key": "auto_scaling_cluster",
                "new_cluster": {
                    "autoscale": {"max_workers": 16, "min_workers": 2},
                    "aws_attributes": {"availability": "SPOT", "zone_id": "us-west-2a"},
                    "node_type_id": "i3.xlarge",
                    "spark_conf": {"spark.speculation": True},
                    "spark_version": "7.3.x-scala2.12",
                },
            }
        ],
        max_items=100,
    )
    max_concurrent_runs: Optional[int] = Field(
        None,
        description=(
            "An optional maximum allowed number of concurrent runs of the job.\n\nSet"
            " this value if you want to be able to execute multiple runs of the same"
            " job concurrently. This is useful for example if you trigger your job on a"
            " frequent schedule and want to allow consecutive runs to overlap with each"
            " other, or if you want to trigger multiple runs which differ by their"
            " input parameters.\n\nThis setting affects only new runs. For example,"
            " suppose the job’s concurrency is 4 and there are 4 concurrent active"
            " runs. Then setting the concurrency to 3 won’t kill any of the active"
            " runs. However, from then on, new runs are skipped unless there are fewer"
            " than 3 active runs.\n\nThis value cannot exceed 1000\\. Setting this"
            " value to 0 causes all new runs to be skipped. The default behavior is to"
            " allow only 1 concurrent run."
        ),
        example=10,
    )
    name: Optional[str] = Field(
        "Untitled",
        description="An optional name for the job.",
        example="A multitask job",
    )
    schedule: Optional[CronSchedule] = Field(
        None,
        description=(
            "An optional periodic schedule for this job. The default behavior is that"
            " the job only runs when triggered by clicking “Run Now” in the Jobs UI or"
            " sending an API request to `runNow`."
        ),
    )
    tags: Optional[Dict[str, Any]] = Field(
        "{}",
        description=(
            "A map of tags associated with the job. These are forwarded to the cluster"
            " as cluster tags for jobs clusters, and are subject to the same"
            " limitations as cluster tags. A maximum of 25 tags can be added to the"
            " job."
        ),
        example={"cost-center": "engineering", "team": "jobs"},
    )
    tasks: Optional[List[JobTaskSettings]] = Field(
        None,
        description="A list of task specifications to be executed by this job.",
        example=[
            {
                "depends_on": [],
                "description": "Extracts session data from events",
                "existing_cluster_id": "0923-164208-meows279",
                "libraries": [{"jar": "dbfs:/mnt/databricks/Sessionize.jar"}],
                "max_retries": 3,
                "min_retry_interval_millis": 2000,
                "retry_on_timeout": False,
                "spark_jar_task": {
                    "main_class_name": "com.databricks.Sessionize",
                    "parameters": ["--data", "dbfs:/path/to/data.json"],
                },
                "task_key": "Sessionize",
                "timeout_seconds": 86400,
            },
            {
                "depends_on": [],
                "description": "Ingests order data",
                "job_cluster_key": "auto_scaling_cluster",
                "libraries": [{"jar": "dbfs:/mnt/databricks/OrderIngest.jar"}],
                "max_retries": 3,
                "min_retry_interval_millis": 2000,
                "retry_on_timeout": False,
                "spark_jar_task": {
                    "main_class_name": "com.databricks.OrdersIngest",
                    "parameters": ["--data", "dbfs:/path/to/order-data.json"],
                },
                "task_key": "Orders_Ingest",
                "timeout_seconds": 86400,
            },
            {
                "depends_on": [
                    {"task_key": "Orders_Ingest"},
                    {"task_key": "Sessionize"},
                ],
                "description": "Matches orders with user sessions",
                "max_retries": 3,
                "min_retry_interval_millis": 2000,
                "new_cluster": {
                    "autoscale": {"max_workers": 16, "min_workers": 2},
                    "aws_attributes": {"availability": "SPOT", "zone_id": "us-west-2a"},
                    "node_type_id": "i3.xlarge",
                    "spark_conf": {"spark.speculation": True},
                    "spark_version": "7.3.x-scala2.12",
                },
                "notebook_task": {
                    "base_parameters": {"age": "35", "name": "John Doe"},
                    "notebook_path": "/Users/user.name@databricks.com/Match",
                    "source": "WORKSPACE",
                },
                "retry_on_timeout": False,
                "task_key": "Match",
                "timeout_seconds": 86400,
            },
        ],
        max_items=100,
    )
    timeout_seconds: Optional[int] = Field(
        None,
        description=(
            "An optional timeout applied to each run of this job. The default behavior"
            " is to have no timeout."
        ),
        example=86400,
    )
    webhook_notifications: Optional[WebhookNotifications] = Field(
        None,
        description=(
            "A collection of system notification IDs to notify when runs of this job"
            " begin or complete. The default behavior is to not send any system"
            " notifications."
        ),
    )

JobTask

Bases: BaseModel

See source code for the fields' description.

Source code in prefect_databricks/models/jobs.py

class JobTask(BaseModel):
    """
    See source code for the fields' description.
    """

    class Config:
        extra = Extra.allow
        allow_mutation = False

    dbt_task: Optional[DbtTask] = Field(
        None,
        description=(
            "If dbt_task, indicates that this must execute a dbt task. It requires both"
            " Databricks SQL and the ability to use a serverless or a pro SQL"
            " warehouse."
        ),
    )
    notebook_task: Optional[NotebookTask] = Field(
        None,
        description=(
            "If notebook_task, indicates that this job must run a notebook. This field"
            " may not be specified in conjunction with spark_jar_task."
        ),
    )
    pipeline_task: Optional[PipelineTask] = Field(
        None,
        description=(
            "If pipeline_task, indicates that this job must execute a Pipeline."
        ),
    )
    python_wheel_task: Optional[PythonWheelTask] = Field(
        None,
        description=(
            "If python_wheel_task, indicates that this job must execute a PythonWheel."
        ),
    )
    spark_jar_task: Optional[SparkJarTask] = Field(
        None,
        description="If spark_jar_task, indicates that this job must run a JAR.",
        example="",
    )
    spark_python_task: Optional[SparkPythonTask] = Field(
        None,
        description=(
            "If spark_python_task, indicates that this job must run a Python file."
        ),
    )
    spark_submit_task: Optional[SparkSubmitTask] = Field(
        None,
        description=(
            "If spark_submit_task, indicates that this job must be launched by the"
            " spark submit script."
        ),
    )
    sql_task: Optional[SqlTask] = Field(
        None,
        description=(
            "If sql_task, indicates that this job must execute a SQL task. It requires"
            " both Databricks SQL and a serverless or a pro SQL warehouse."
        ),
    )

JobTaskSettings

Bases: BaseModel

See source code for the fields' description.

Source code in prefect_databricks/models/jobs.py

class JobTaskSettings(BaseModel):
    """
    See source code for the fields' description.
    """

    class Config:
        extra = Extra.allow
        allow_mutation = False

    dbt_task: Optional[DbtTask] = Field(
        None,
        description=(
            "If dbt_task, indicates that this must execute a dbt task. It requires both"
            " Databricks SQL and the ability to use a serverless or a pro SQL"
            " warehouse."
        ),
    )
    depends_on: Optional[TaskDependencies] = None
    description: Optional[TaskDescription] = None
    email_notifications: Optional[JobEmailNotifications] = Field(
        None,
        description=(
            "An optional set of email addresses that is notified when runs of this task"
            " begin or complete as well as when this task is deleted. The default"
            " behavior is to not send any emails."
        ),
    )
    existing_cluster_id: Optional[str] = Field(
        None,
        description=(
            "If existing_cluster_id, the ID of an existing cluster that is used for all"
            " runs of this task. When running tasks on an existing cluster, you may"
            " need to manually restart the cluster if it stops responding. We suggest"
            " running jobs on new clusters for greater reliability."
        ),
        example="0923-164208-meows279",
    )
    job_cluster_key: Optional[str] = Field(
        None,
        description=(
            "If job_cluster_key, this task is executed reusing the cluster specified in"
            " `job.settings.job_clusters`."
        ),
        max_length=100,
        min_length=1,
        regex="^[\\w\\-]+$",
    )
    libraries: Optional[List[Library]] = Field(
        None,
        description=(
            "An optional list of libraries to be installed on the cluster that executes"
            " the task. The default value is an empty list."
        ),
    )
    max_retries: Optional[int] = Field(
        None,
        description=(
            "An optional maximum number of times to retry an unsuccessful run. A run is"
            " considered to be unsuccessful if it completes with the `FAILED`"
            " result_state or `INTERNAL_ERROR` `life_cycle_state`. The value -1 means"
            " to retry indefinitely and the value 0 means to never retry. The default"
            " behavior is to never retry."
        ),
        example=10,
    )
    min_retry_interval_millis: Optional[int] = Field(
        None,
        description=(
            "An optional minimal interval in milliseconds between the start of the"
            " failed run and the subsequent retry run. The default behavior is that"
            " unsuccessful runs are immediately retried."
        ),
        example=2000,
    )
    new_cluster: Optional[NewCluster] = Field(
        None,
        description=(
            "If new_cluster, a description of a cluster that is created for each run."
        ),
    )
    notebook_task: Optional[NotebookTask] = Field(
        None,
        description=(
            "If notebook_task, indicates that this task must run a notebook. This field"
            " may not be specified in conjunction with spark_jar_task."
        ),
    )
    pipeline_task: Optional[PipelineTask] = Field(
        None,
        description=(
            "If pipeline_task, indicates that this task must execute a Pipeline."
        ),
    )
    python_wheel_task: Optional[PythonWheelTask] = Field(
        None,
        description=(
            "If python_wheel_task, indicates that this job must execute a PythonWheel."
        ),
    )
    retry_on_timeout: Optional[bool] = Field(
        None,
        description=(
            "An optional policy to specify whether to retry a task when it times out."
            " The default behavior is to not retry on timeout."
        ),
        example=True,
    )
    spark_jar_task: Optional[SparkJarTask] = Field(
        None, description="If spark_jar_task, indicates that this task must run a JAR."
    )
    spark_python_task: Optional[SparkPythonTask] = Field(
        None,
        description=(
            "If spark_python_task, indicates that this task must run a Python file."
        ),
    )
    spark_submit_task: Optional[SparkSubmitTask] = Field(
        None,
        description=(
            "If spark_submit_task, indicates that this task must be launched by the"
            " spark submit script."
        ),
    )
    sql_task: Optional[SqlTask] = Field(
        None,
        description=(
            "If sql_task, indicates that this job must execute a SQL task. It requires"
            " both Databricks SQL and a serverless or a pro SQL warehouse."
        ),
    )
    task_key: TaskKey
    timeout_seconds: Optional[int] = Field(
        None,
        description=(
            "An optional timeout applied to each run of this job task. The default"
            " behavior is to have no timeout."
        ),
        example=86400,
    )
    webhook_notifications: Optional[WebhookNotifications] = Field(
        None,
        description=(
            "A collection of system notification IDs to notify when the run begins or"
            " completes. The default behavior is to not send any system notifications."
        ),
    )

Library

Bases: BaseModel

See source code for the fields' description.

Source code in prefect_databricks/models/jobs.py

class Library(BaseModel):
    """
    See source code for the fields' description.
    """

    class Config:
        extra = Extra.allow
        allow_mutation = False

    cran: Optional[RCranLibrary] = Field(
        None, description="If cran, specification of a CRAN library to be installed."
    )
    egg: Optional[str] = Field(
        None,
        description=(
            "If egg, URI of the egg to be installed. DBFS and S3 URIs are supported."
            ' For example: `{ "egg": "dbfs:/my/egg" }` or `{ "egg":'
            ' "s3://my-bucket/egg" }`. If S3 is used, make sure the cluster has read'
            " access on the library. You may need to launch the cluster with an"
            " instance profile to access the S3 URI."
        ),
        example="dbfs:/my/egg",
    )
    jar: Optional[str] = Field(
        None,
        description=(
            "If jar, URI of the JAR to be installed. DBFS and S3 URIs are supported."
            ' For example: `{ "jar": "dbfs:/mnt/databricks/library.jar" }` or `{ "jar":'
            ' "s3://my-bucket/library.jar" }`. If S3 is used, make sure the cluster has'
            " read access on the library. You may need to launch the cluster with an"
            " instance profile to access the S3 URI."
        ),
        example="dbfs:/my-jar.jar",
    )
    maven: Optional[MavenLibrary] = Field(
        None,
        description=(
            "If maven, specification of a Maven library to be installed. For example:"
            ' `{ "coordinates": "org.jsoup:jsoup:1.7.2" }`'
        ),
    )
    pypi: Optional[PythonPyPiLibrary] = Field(
        None,
        description=(
            "If pypi, specification of a PyPI library to be installed. Specifying the"
            " `repo` field is optional and if not specified, the default pip index is"
            ' used. For example: `{ "package": "simplejson", "repo":'
            ' "https://my-repo.com" }`'
        ),
    )
    whl: Optional[str] = Field(
        None,
        description=(
            "If whl, URI of the wheel or zipped wheels to be installed. DBFS and S3"
            ' URIs are supported. For example: `{ "whl": "dbfs:/my/whl" }` or `{ "whl":'
            ' "s3://my-bucket/whl" }`. If S3 is used, make sure the cluster has read'
            " access on the library. You may need to launch the cluster with an"
            " instance profile to access the S3 URI. Also the wheel file name needs to"
            " use the [correct"
            " convention](https://www.python.org/dev/peps/pep-0427/#file-format). If"
            " zipped wheels are to be installed, the file name suffix should be"
            " `.wheelhouse.zip`."
        ),
        example="dbfs:/my/whl",
    )

LibraryFullStatus

Bases: BaseModel

See source code for the fields' description.

Source code in prefect_databricks/models/jobs.py

class LibraryFullStatus(BaseModel):
    """
    See source code for the fields' description.
    """

    class Config:
        extra = Extra.allow
        allow_mutation = False

    is_library_for_all_clusters: Optional[bool] = Field(
        None,
        description=(
            "Whether the library was set to be installed on all clusters via the"
            " libraries UI."
        ),
    )
    library: Optional[Library] = Field(
        None, description="Unique identifier for the library."
    )
    messages: Optional[List[str]] = Field(
        None,
        description=(
            "All the info and warning messages that have occurred so far for this"
            " library."
        ),
    )
    status: Optional[LibraryInstallStatus] = Field(
        None, description="Status of installing the library on the cluster."
    )

LibraryInstallStatus

Bases: str, Enum

* `PENDING`: No action has yet been taken to install the library. This state should be very short lived.

• RESOLVING: Metadata necessary to install the library is being retrieved from the provided repository. For Jar, Egg, and Whl libraries, this step is a no-op.
• INSTALLING: The library is actively being installed, either by adding resources to Spark or executing system commands inside the Spark nodes.
• INSTALLED: The library has been successfully instally.
• SKIPPED: Installation on a Databricks Runtime 7.0 or above cluster was skipped due to Scala version incompatibility.
• FAILED: Some step in installation failed. More information can be found in the messages field.
• UNINSTALL_ON_RESTART: The library has been marked for removal. Libraries can be removed only when clusters are restarted, so libraries that enter this state remains until the cluster is restarted.

Source code in prefect_databricks/models/jobs.py

class LibraryInstallStatus(str, Enum):
    """
        * `PENDING`: No action has yet been taken to install the library. This state should be very short lived.
    * `RESOLVING`: Metadata necessary to install the library is being retrieved from the provided repository. For Jar, Egg, and Whl libraries, this step is a no-op.
    * `INSTALLING`: The library is actively being installed, either by adding resources to Spark or executing system commands inside the Spark nodes.
    * `INSTALLED`: The library has been successfully instally.
    * `SKIPPED`: Installation on a Databricks Runtime 7.0 or above cluster was skipped due to Scala version incompatibility.
    * `FAILED`: Some step in installation failed. More information can be found in the messages field.
    * `UNINSTALL_ON_RESTART`: The library has been marked for removal. Libraries can be removed only when clusters are restarted, so libraries that enter this state remains until the cluster is restarted.
    """

    pending = "PENDING"
    resolving = "RESOLVING"
    installing = "INSTALLING"
    installed = "INSTALLED"
    skipped = "SKIPPED"
    failed = "FAILED"
    uninstallonrestart = "UNINSTALL_ON_RESTART"

ListOrder

Bases: str, Enum

* `DESC`: Descending order.

• ASC: Ascending order.

Source code in prefect_databricks/models/jobs.py

class ListOrder(str, Enum):
    """
        * `DESC`: Descending order.
    * `ASC`: Ascending order.
    """

    desc = "DESC"
    asc = "ASC"

LogSyncStatus

Bases: BaseModel

See source code for the fields' description.

Source code in prefect_databricks/models/jobs.py

class LogSyncStatus(BaseModel):
    """
    See source code for the fields' description.
    """

    class Config:
        extra = Extra.allow
        allow_mutation = False

    last_attempted: Optional[int] = Field(
        None,
        description=(
            "The timestamp of last attempt. If the last attempt fails, last_exception"
            " contains the exception in the last attempt."
        ),
    )
    last_exception: Optional[str] = Field(
        None,
        description=(
            "The exception thrown in the last attempt, it would be null (omitted in the"
            " response) if there is no exception in last attempted."
        ),
    )

MavenLibrary

Bases: BaseModel

See source code for the fields' description.

Source code in prefect_databricks/models/jobs.py

class MavenLibrary(BaseModel):
    """
    See source code for the fields' description.
    """

    class Config:
        extra = Extra.allow
        allow_mutation = False

    coordinates: str = Field(
        ...,
        description=(
            "Gradle-style Maven coordinates. For example: `org.jsoup:jsoup:1.7.2`. This"
            " field is required."
        ),
        example="org.jsoup:jsoup:1.7.2",
    )
    exclusions: Optional[List[str]] = Field(
        None,
        description=(
            'List of dependences to exclude. For example: `["slf4j:slf4j",'
            ' "*:hadoop-client"]`.\n\nMaven dependency exclusions:'
            " <https://maven.apache.org/guides/introduction/introduction-to-optional-and-excludes-dependencies.html>."
        ),
        example=["slf4j:slf4j", "*:hadoop-client"],
    )
    repo: Optional[str] = Field(
        None,
        description=(
            "Maven repo to install the Maven package from. If omitted, both Maven"
            " Central Repository and Spark Packages are searched."
        ),
        example="https://my-repo.com",
    )

NewCluster

Bases: BaseModel

See source code for the fields' description.

Source code in prefect_databricks/models/jobs.py

class NewCluster(BaseModel):
    """
    See source code for the fields' description.
    """

    class Config:
        extra = Extra.allow
        allow_mutation = False

    autoscale: Optional[AutoScale] = Field(
        None,
        description=(
            "If autoscale, the required parameters to automatically scale clusters up"
            " and down based on load."
        ),
    )
    aws_attributes: Optional[AwsAttributes] = Field(
        None,
        description=(
            "Attributes related to clusters running on Amazon Web Services. If not"
            " specified at cluster creation, a set of default values is used."
        ),
    )
    cluster_log_conf: Optional[ClusterLogConf] = Field(
        None,
        description=(
            "The configuration for delivering Spark logs to a long-term storage"
            " destination. Only one destination can be specified for one cluster. If"
            " the conf is given, the logs are delivered to the destination every `5"
            " mins`. The destination of driver logs is"
            " `<destination>/<cluster-id>/driver`, while the destination of executor"
            " logs is `<destination>/<cluster-id>/executor`."
        ),
    )
    custom_tags: Optional[ClusterTag] = Field(
        None,
        description=(
            "An object containing a set of tags for cluster resources. Databricks tags"
            " all cluster resources (such as AWS instances and EBS volumes) with these"
            " tags in addition to default_tags.\n\n**Note**:\n\n* Tags are not"
            " supported on legacy node types such as compute-optimized and"
            " memory-optimized\n* Databricks allows at most 45 custom tags"
        ),
    )
    driver_instance_pool_id: Optional[str] = Field(
        None,
        description=(
            "The optional ID of the instance pool to use for the driver node. You must"
            " also specify `instance_pool_id`. Refer to [Instance Pools"
            " API](https://docs.databricks.com/dev-tools/api/latest/instance-pools.html)"
            " for details."
        ),
    )
    driver_node_type_id: Optional[str] = Field(
        None,
        description=(
            "The node type of the Spark driver. This field is optional; if unset, the"
            " driver node type is set as the same value as `node_type_id` defined"
            " above."
        ),
    )
    enable_elastic_disk: Optional[bool] = Field(
        None,
        description=(
            "Autoscaling Local Storage: when enabled, this cluster dynamically acquires"
            " additional disk space when its Spark workers are running low on disk"
            " space. This feature requires specific AWS permissions to function"
            " correctly - refer to [Autoscaling local"
            " storage](https://docs.databricks.com/clusters/configure.html#autoscaling-local-storage)"
            " for details."
        ),
    )
    enable_local_disk_encryption: Optional[bool] = Field(
        None,
        description=(
            "Determines whether encryption of disks locally attached to the cluster is"
            " enabled."
        ),
    )
    init_scripts: Optional[List[InitScriptInfo]] = Field(
        None,
        description=(
            "The configuration for storing init scripts. Any number of scripts can be"
            " specified. The scripts are executed sequentially in the order provided."
            " If `cluster_log_conf` is specified, init script logs are sent to"
            " `<destination>/<cluster-id>/init_scripts`."
        ),
    )
    instance_pool_id: Optional[str] = Field(
        None,
        description=(
            "The optional ID of the instance pool to use for cluster nodes. If"
            " `driver_instance_pool_id` is present, `instance_pool_id` is used for"
            " worker nodes only. Otherwise, it is used for both the driver node and"
            " worker nodes. Refer to [Instance Pools"
            " API](https://docs.databricks.com/dev-tools/api/latest/instance-pools.html)"
            " for details."
        ),
    )
    node_type_id: Optional[str] = Field(
        None,
        description=(
            "This field encodes, through a single value, the resources available to"
            " each of the Spark nodes in this cluster. For example, the Spark nodes can"
            " be provisioned and optimized for memory or compute intensive workloads A"
            " list of available node types can be retrieved by using the [List node"
            " types](https://docs.databricks.com/dev-tools/api/latest/clusters.html#list-node-types)"
            " API call."
        ),
    )
    num_workers: Optional[int] = Field(
        None,
        description=(
            "If num_workers, number of worker nodes that this cluster must have. A"
            " cluster has one Spark driver and num_workers executors for a total of"
            " num_workers + 1 Spark nodes. When reading the properties of a cluster,"
            " this field reflects the desired number of workers rather than the actual"
            " current number of workers. For example, if a cluster is resized from 5 to"
            " 10 workers, this field immediately updates to reflect the target size of"
            " 10 workers, whereas the workers listed in `spark_info` gradually increase"
            " from 5 to 10 as the new nodes are provisioned."
        ),
    )
    policy_id: Optional[str] = Field(
        None,
        description=(
            "A [cluster"
            " policy](https://docs.databricks.com/dev-tools/api/latest/policies.html)"
            " ID. Either `node_type_id` or `instance_pool_id` must be specified in the"
            " cluster policy if they are not specified in this job cluster object."
        ),
    )
    spark_conf: Optional[SparkConfPair] = Field(
        None,
        description=(
            "An object containing a set of optional, user-specified Spark configuration"
            " key-value pairs. You can also pass in a string of extra JVM options to"
            " the driver and the executors via `spark.driver.extraJavaOptions` and"
            " `spark.executor.extraJavaOptions` respectively.\n\nExample Spark confs:"
            ' `{"spark.speculation": true, "spark.streaming.ui.retainedBatches": 5}` or'
            ' `{"spark.driver.extraJavaOptions": "-verbose:gc -XX:+PrintGCDetails"}`'
        ),
    )
    spark_env_vars: Optional[SparkEnvPair] = Field(
        None,
        description=(
            "An object containing a set of optional, user-specified environment"
            " variable key-value pairs. Key-value pair of the form (X,Y) are exported"
            " as is (for example, `export X='Y'`) while launching the driver and"
            " workers.\n\nTo specify an additional set of `SPARK_DAEMON_JAVA_OPTS`, we"
            " recommend appending them to `$SPARK_DAEMON_JAVA_OPTS` as shown in the"
            " following example. This ensures that all default databricks managed"
            " environmental variables are included as well.\n\nExample Spark"
            ' environment variables: `{"SPARK_WORKER_MEMORY": "28000m",'
            ' "SPARK_LOCAL_DIRS": "/local_disk0"}` or `{"SPARK_DAEMON_JAVA_OPTS":'
            ' "$SPARK_DAEMON_JAVA_OPTS -Dspark.shuffle.service.enabled=true"}`'
        ),
    )
    spark_version: str = Field(
        ...,
        description=(
            "The Spark version of the cluster. A list of available Spark versions can"
            " be retrieved by using the [Runtime"
            " versions](https://docs.databricks.com/dev-tools/api/latest/clusters.html#runtime-versions)"
            " API call."
        ),
    )
    runtime_engine: Optional[RuntimeEngine] = Field(
        None,
        description=(
            "Decides which runtime engine to be use, e.g. Standard vs. Photon. If "
            "unspecified, the runtime engine is inferred from spark_version. "
            "see https://docs.databricks.com/api-explorer/workspace/jobs/create "
            " for more details"
        ),
    )
    ssh_public_keys: Optional[List[str]] = Field(
        None,
        description=(
            "SSH public key contents that are added to each Spark node in this cluster."
            " The corresponding private keys can be used to login with the user name"
            " `ubuntu` on port `2200`. Up to 10 keys can be specified."
        ),
    )

NodeType

Bases: BaseModel

See source code for the fields' description.

Source code in prefect_databricks/models/jobs.py

class NodeType(BaseModel):
    """
    See source code for the fields' description.
    """

    class Config:
        extra = Extra.allow
        allow_mutation = False

    description: str = Field(
        ...,
        description=(
            "A string description associated with this node type. This field is"
            " required."
        ),
    )
    instance_type_id: str = Field(
        ...,
        description=(
            "An identifier for the type of hardware that this node runs on. This field"
            " is required."
        ),
    )
    is_deprecated: Optional[bool] = Field(
        None,
        description=(
            "Whether the node type is deprecated. Non-deprecated node types offer"
            " greater performance."
        ),
    )
    memory_mb: int = Field(
        ...,
        description=(
            "Memory (in MB) available for this node type. This field is required."
        ),
    )
    node_info: Optional[ClusterCloudProviderNodeInfo] = Field(
        None, description="Node type info reported by the cloud provider."
    )
    node_type_id: str = Field(
        ..., description="Unique identifier for this node type. This field is required."
    )
    num_cores: Optional[float] = Field(
        None,
        description=(
            "Number of CPU cores available for this node type. This can be fractional"
            " if the number of cores on a machine instance is not divisible by the"
            " number of Spark nodes on that machine. This field is required."
        ),
    )

NotebookOutput

Bases: BaseModel

See source code for the fields' description.

Source code in prefect_databricks/models/jobs.py

class NotebookOutput(BaseModel):
    """
    See source code for the fields' description.
    """

    class Config:
        extra = Extra.allow
        allow_mutation = False

    result: Optional[str] = Field(
        None,
        description=(
            "The value passed to"
            " [dbutils.notebook.exit()](https://docs.databricks.com/notebooks/notebook-workflows.html#notebook-workflows-exit)."
            " Databricks restricts this API to return the first 5 MB of the value. For"
            " a larger result, your job can store the results in a cloud storage"
            " service. This field is absent if `dbutils.notebook.exit()` was never"
            " called."
        ),
        example="An arbitrary string passed by calling dbutils.notebook.exit(...)",
    )
    truncated: Optional[bool] = Field(
        None, description="Whether or not the result was truncated.", example=False
    )

NotebookTask

Bases: BaseModel

See source code for the fields' description.

Source code in prefect_databricks/models/jobs.py

class NotebookTask(BaseModel):
    """
    See source code for the fields' description.
    """

    class Config:
        extra = Extra.allow
        allow_mutation = False

    base_parameters: Optional[Dict[str, Any]] = Field(
        None,
        description=(
            "Base parameters to be used for each run of this job. If the run is"
            " initiated by a call to"
            " [`run-now`](https://docs.databricks.com/dev-tools/api/latest/jobs.html#operation/JobsRunNow)"
            " with parameters specified, the two parameters maps are merged. If the"
            " same key is specified in `base_parameters` and in `run-now`, the value"
            " from `run-now` is used.\n\nUse [Task parameter"
            " variables](https://docs.databricks.com/jobs.html#parameter-variables) to"
            " set parameters containing information about job runs.\n\nIf the notebook"
            " takes a parameter that is not specified in the job’s `base_parameters` or"
            " the `run-now` override parameters, the default value from the notebook is"
            " used.\n\nRetrieve these parameters in a notebook using"
            " [dbutils.widgets.get](https://docs.databricks.com/dev-tools/databricks-utils.html#dbutils-widgets)."
        ),
        example={"age": 35, "name": "John Doe"},
    )
    notebook_path: str = Field(
        ...,
        description=(
            "The path of the notebook to be run in the Databricks workspace or remote"
            " repository. For notebooks stored in the Databricks workspace, the path"
            " must be absolute and begin with a slash. For notebooks stored in a remote"
            " repository, the path must be relative. This field is required."
        ),
        example="/Users/user.name@databricks.com/notebook_to_run",
    )
    source: Optional[Literal["WORKSPACE", "GIT"]] = Field(
        None,
        description=(
            "Optional location type of the notebook. When set to `WORKSPACE`, the"
            " notebook will be retrieved from the local Databricks workspace. When set"
            " to `GIT`, the notebook will be retrieved from a Git repository defined in"
            " `git_source`. If the value is empty, the task will use `GIT` if"
            " `git_source` is defined and `WORKSPACE` otherwise."
        ),
        example="WORKSPACE",
    )

OnFailureItem

Bases: BaseModel

See source code for the fields' description.

Source code in prefect_databricks/models/jobs.py

class OnFailureItem(BaseModel):
    """
    See source code for the fields' description.
    """

    class Config:
        extra = Extra.allow
        allow_mutation = False

    id: Optional[str] = None

OnStartItem

Bases: BaseModel

See source code for the fields' description.

Source code in prefect_databricks/models/jobs.py

class OnStartItem(BaseModel):
    """
    See source code for the fields' description.
    """

    class Config:
        extra = Extra.allow
        allow_mutation = False

    id: Optional[str] = None

OnSucces

Bases: BaseModel

See source code for the fields' description.

Source code in prefect_databricks/models/jobs.py

class OnSucces(BaseModel):
    """
    See source code for the fields' description.
    """

    class Config:
        extra = Extra.allow
        allow_mutation = False

    id: Optional[str] = None

ParameterPair

Bases: BaseModel

See source code for the fields' description.

An object with additional information about why a cluster was terminated. The object keys are one of TerminationParameter and the value is the termination information.

Source code in prefect_databricks/models/jobs.py

class ParameterPair(BaseModel):
    """
    See source code for the fields' description.

    An object with additional information about why a cluster was terminated. The object keys are one of `TerminationParameter` and the value is the termination information.
    """

    class Config:
        extra = Extra.allow

        allow_mutation = False

PermissionLevel

Bases: BaseModel

See source code for the fields' description.

Source code in prefect_databricks/models/jobs.py

class PermissionLevel(BaseModel):
    """
    See source code for the fields' description.
    """

    class Config:
        extra = Extra.allow
        allow_mutation = False

    __root__: Union[CanManage, CanManageRun, CanView, IsOwner] = Field(
        ..., description="Permission level to grant."
    )

PermissionLevelForGroup

Bases: BaseModel

See source code for the fields' description.

Source code in prefect_databricks/models/jobs.py

class PermissionLevelForGroup(BaseModel):
    """
    See source code for the fields' description.
    """

    class Config:
        extra = Extra.allow
        allow_mutation = False

    __root__: Union[CanManage, CanManageRun, CanView] = Field(
        ..., description="Permission level to grant."
    )

PipelineParams

Bases: BaseModel

See source code for the fields' description.

Source code in prefect_databricks/models/jobs.py

class PipelineParams(BaseModel):
    """
    See source code for the fields' description.
    """

    class Config:
        extra = Extra.allow
        allow_mutation = False

    full_refresh: Optional[bool] = Field(
        None, description="If true, triggers a full refresh on the delta live table."
    )

PipelineTask

Bases: BaseModel

See source code for the fields' description.

Source code in prefect_databricks/models/jobs.py

class PipelineTask(BaseModel):
    """
    See source code for the fields' description.
    """

    class Config:
        extra = Extra.allow
        allow_mutation = False

    full_refresh: Optional[bool] = Field(
        False,
        description=(
            "If true, a full refresh will be triggered on the delta live table."
        ),
    )
    pipeline_id: Optional[str] = Field(
        None,
        description="The full name of the pipeline task to execute.",
        example="a12cd3e4-0ab1-1abc-1a2b-1a2bcd3e4fg5",
    )

PoolClusterTerminationCode

Bases: str, Enum

* INSTANCE_POOL_MAX_CAPACITY_FAILURE: The pool max capacity has been reached.

• INSTANCE_POOL_NOT_FOUND_FAILURE: The pool specified by the cluster is no longer active or doesn’t exist.

Source code in prefect_databricks/models/jobs.py

class PoolClusterTerminationCode(str, Enum):
    """
        * INSTANCE_POOL_MAX_CAPACITY_FAILURE: The pool max capacity has been reached.
    * INSTANCE_POOL_NOT_FOUND_FAILURE: The pool specified by the cluster is no longer active or doesn’t exist.
    """

    instancepoolmaxcapacityfailure = "INSTANCE_POOL_MAX_CAPACITY_FAILURE"
    instancepoolnotfoundfailure = "INSTANCE_POOL_NOT_FOUND_FAILURE"

PythonPyPiLibrary

Bases: BaseModel

See source code for the fields' description.

Source code in prefect_databricks/models/jobs.py

class PythonPyPiLibrary(BaseModel):
    """
    See source code for the fields' description.
    """

    class Config:
        extra = Extra.allow
        allow_mutation = False

    package: str = Field(
        ...,
        description=(
            "The name of the PyPI package to install. An optional exact version"
            " specification is also supported. Examples: `simplejson` and"
            " `simplejson==3.8.0`. This field is required."
        ),
        example="simplejson==3.8.0",
    )
    repo: Optional[str] = Field(
        None,
        description=(
            "The repository where the package can be found. If not specified, the"
            " default pip index is used."
        ),
        example="https://my-repo.com",
    )

PythonWheelTask

Bases: BaseModel

See source code for the fields' description.

Source code in prefect_databricks/models/jobs.py

class PythonWheelTask(BaseModel):
    """
    See source code for the fields' description.
    """

    class Config:
        extra = Extra.allow
        allow_mutation = False

    entry_point: Optional[str] = Field(
        None,
        description=(
            "Named entry point to use, if it does not exist in the metadata of the"
            " package it executes the function from the package directly using"
            " `$packageName.$entryPoint()`"
        ),
    )
    named_parameters: Optional[Dict[str, Any]] = Field(
        None,
        description=(
            "Command-line parameters passed to Python wheel task in the form of"
            ' `["--name=task", "--data=dbfs:/path/to/data.json"]`. Leave it empty if'
            " `parameters` is not null."
        ),
        example={"data": "dbfs:/path/to/data.json", "name": "task"},
    )
    package_name: Optional[str] = Field(
        None, description="Name of the package to execute"
    )
    parameters: Optional[List[str]] = Field(
        None,
        description=(
            "Command-line parameters passed to Python wheel task. Leave it empty if"
            " `named_parameters` is not null."
        ),
        example=["--name=task", "one", "two"],
    )

RCranLibrary

Bases: BaseModel

See source code for the fields' description.

Source code in prefect_databricks/models/jobs.py

class RCranLibrary(BaseModel):
    """
    See source code for the fields' description.
    """

    class Config:
        extra = Extra.allow
        allow_mutation = False

    package: str = Field(
        ...,
        description="The name of the CRAN package to install. This field is required.",
        example="geojson",
    )
    repo: Optional[str] = Field(
        None,
        description=(
            "The repository where the package can be found. If not specified, the"
            " default CRAN repo is used."
        ),
        example="https://my-repo.com",
    )

RepairHistory

Bases: BaseModel

See source code for the fields' description.

Source code in prefect_databricks/models/jobs.py

class RepairHistory(BaseModel):
    """
    See source code for the fields' description.
    """

    class Config:
        extra = Extra.allow
        allow_mutation = False

    repair_history: Optional[List[RepairHistoryItem]] = Field(
        None, description="The repair history of the run."
    )

RepairHistoryItem

Bases: BaseModel

See source code for the fields' description.

Source code in prefect_databricks/models/jobs.py

class RepairHistoryItem(BaseModel):
    """
    See source code for the fields' description.
    """

    class Config:
        extra = Extra.allow
        allow_mutation = False

    end_time: Optional[int] = Field(
        None, description="The end time of the (repaired) run.", example=1625060863413
    )
    id: Optional[int] = Field(
        None,
        description=(
            "The ID of the repair. Only returned for the items that represent a repair"
            " in `repair_history`."
        ),
        example=734650698524280,
    )
    start_time: Optional[int] = Field(
        None, description="The start time of the (repaired) run.", example=1625060460483
    )
    state: Optional[RunState] = None
    task_run_ids: Optional[List[int]] = Field(
        None,
        description=(
            "The run IDs of the task runs that ran as part of this repair history item."
        ),
        example=[1106460542112844, 988297789683452],
    )
    type: Optional[Literal["ORIGINAL", "REPAIR"]] = Field(
        None,
        description=(
            "The repair history item type. Indicates whether a run is the original run"
            " or a repair run."
        ),
    )

RepairRunInput

Bases: BaseModel

See source code for the fields' description.

Source code in prefect_databricks/models/jobs.py

class RepairRunInput(BaseModel):
    """
    See source code for the fields' description.
    """

    class Config:
        extra = Extra.allow
        allow_mutation = False

    latest_repair_id: Optional[int] = Field(
        None,
        description=(
            "The ID of the latest repair. This parameter is not required when repairing"
            " a run for the first time, but must be provided on subsequent requests to"
            " repair the same run."
        ),
        example=734650698524280,
    )
    rerun_all_failed_tasks: Optional[bool] = Field(
        False,
        description=(
            "If true, repair all failed tasks. Only one of rerun_tasks or"
            " rerun_all_failed_tasks can be used."
        ),
    )
    rerun_tasks: Optional[List[str]] = Field(
        None,
        description="The task keys of the task runs to repair.",
        example=["task0", "task1"],
    )
    run_id: Optional[int] = Field(
        None,
        description=(
            "The job run ID of the run to repair. The run must not be in progress."
        ),
        example=455644833,
    )

ResizeCause

Bases: str, Enum

* `AUTOSCALE`: Automatically resized based on load.

• USER_REQUEST: User requested a new size.
• AUTORECOVERY: Autorecovery monitor resized the cluster after it lost a node.

Source code in prefect_databricks/models/jobs.py

class ResizeCause(str, Enum):
    """
        * `AUTOSCALE`: Automatically resized based on load.
    * `USER_REQUEST`: User requested a new size.
    * `AUTORECOVERY`: Autorecovery monitor resized the cluster after it lost a node.
    """

    autoscale = "AUTOSCALE"
    userrequest = "USER_REQUEST"
    autorecovery = "AUTORECOVERY"

Run

Bases: BaseModel

See source code for the fields' description.

Source code in prefect_databricks/models/jobs.py

class Run(BaseModel):
    """
    See source code for the fields' description.
    """

    class Config:
        extra = Extra.allow
        allow_mutation = False

    attempt_number: Optional[int] = Field(
        None,
        description=(
            "The sequence number of this run attempt for a triggered job run. The"
            " initial attempt of a run has an attempt_number of 0\\. If the initial run"
            " attempt fails, and the job has a retry policy (`max_retries` \\> 0),"
            " subsequent runs are created with an `original_attempt_run_id` of the"
            " original attempt’s ID and an incrementing `attempt_number`. Runs are"
            " retried only until they succeed, and the maximum `attempt_number` is the"
            " same as the `max_retries` value for the job."
        ),
        example=0,
    )
    cleanup_duration: Optional[int] = Field(
        None,
        description=(
            "The time in milliseconds it took to terminate the cluster and clean up any"
            " associated artifacts. The total duration of the run is the sum of the"
            " setup_duration, the execution_duration, and the cleanup_duration."
        ),
        example=0,
    )
    cluster_instance: Optional[ClusterInstance] = Field(
        None,
        description=(
            "The cluster used for this run. If the run is specified to use a new"
            " cluster, this field is set once the Jobs service has requested a cluster"
            " for the run."
        ),
    )
    cluster_spec: Optional[ClusterSpec] = Field(
        None,
        description=(
            "A snapshot of the job’s cluster specification when this run was created."
        ),
    )
    creator_user_name: Optional[str] = Field(
        None,
        description=(
            "The creator user name. This field won’t be included in the response if the"
            " user has already been deleted."
        ),
        example="user.name@databricks.com",
    )
    end_time: Optional[int] = Field(
        None,
        description=(
            "The time at which this run ended in epoch milliseconds (milliseconds since"
            " 1/1/1970 UTC). This field is set to 0 if the job is still running."
        ),
        example=1625060863413,
    )
    execution_duration: Optional[int] = Field(
        None,
        description=(
            "The time in milliseconds it took to execute the commands in the JAR or"
            " notebook until they completed, failed, timed out, were cancelled, or"
            " encountered an unexpected error."
        ),
        example=0,
    )
    git_source: Optional[GitSource1] = Field(
        None,
        description=(
            "This functionality is in Public Preview.\n\nAn optional specification for"
            " a remote repository containing the notebooks used by this job's notebook"
            " tasks."
        ),
        example={
            "git_branch": "main",
            "git_provider": "gitHub",
            "git_url": "https://github.com/databricks/databricks-cli",
        },
    )
    job_clusters: Optional[List[JobCluster]] = Field(
        None,
        description=(
            "A list of job cluster specifications that can be shared and reused by"
            " tasks of this job. Libraries cannot be declared in a shared job cluster."
            " You must declare dependent libraries in task settings."
        ),
        example=[
            {
                "job_cluster_key": "auto_scaling_cluster",
                "new_cluster": {
                    "autoscale": {"max_workers": 16, "min_workers": 2},
                    "aws_attributes": {"availability": "SPOT", "zone_id": "us-west-2a"},
                    "node_type_id": "i3.xlarge",
                    "spark_conf": {"spark.speculation": True},
                    "spark_version": "7.3.x-scala2.12",
                },
            }
        ],
        max_items=100,
    )
    job_id: Optional[int] = Field(
        None,
        description="The canonical identifier of the job that contains this run.",
        example=11223344,
    )
    number_in_job: Optional[int] = Field(
        None,
        deprecated=True,
        description=(
            "A unique identifier for this job run. This is set to the same value as"
            " `run_id`."
        ),
        example=455644833,
    )
    original_attempt_run_id: Optional[int] = Field(
        None,
        description=(
            "If this run is a retry of a prior run attempt, this field contains the"
            " run_id of the original attempt; otherwise, it is the same as the run_id."
        ),
        example=455644833,
    )
    overriding_parameters: Optional[RunParameters] = Field(
        None, description="The parameters used for this run."
    )
    run_id: Optional[int] = Field(
        None,
        description=(
            "The canonical identifier of the run. This ID is unique across all runs of"
            " all jobs."
        ),
        example=455644833,
    )
    run_name: Optional[str] = Field(
        "Untitled",
        description=(
            "An optional name for the run. The maximum allowed length is 4096 bytes in"
            " UTF-8 encoding."
        ),
        example="A multitask job run",
    )
    run_page_url: Optional[str] = Field(
        None,
        description="The URL to the detail page of the run.",
        example="https://my-workspace.cloud.databricks.com/#job/11223344/run/123",
    )
    run_type: Optional[RunType] = None
    schedule: Optional[CronSchedule] = Field(
        None,
        description=(
            "The cron schedule that triggered this run if it was triggered by the"
            " periodic scheduler."
        ),
    )
    setup_duration: Optional[int] = Field(
        None,
        description=(
            "The time it took to set up the cluster in milliseconds. For runs that run"
            " on new clusters this is the cluster creation time, for runs that run on"
            " existing clusters this time should be very short."
        ),
        example=0,
    )
    start_time: Optional[int] = Field(
        None,
        description=(
            "The time at which this run was started in epoch milliseconds (milliseconds"
            " since 1/1/1970 UTC). This may not be the time when the job task starts"
            " executing, for example, if the job is scheduled to run on a new cluster,"
            " this is the time the cluster creation call is issued."
        ),
        example=1625060460483,
    )
    state: Optional[RunState] = Field(
        None, description="The result and lifecycle states of the run."
    )
    tasks: Optional[List[RunTask]] = Field(
        None,
        description=(
            "The list of tasks performed by the run. Each task has its own `run_id`"
            " which you can use to call `JobsGetOutput` to retrieve the run resutls."
        ),
        example=[
            {
                "attempt_number": 0,
                "cleanup_duration": 0,
                "cluster_instance": {
                    "cluster_id": "0923-164208-meows279",
                    "spark_context_id": "4348585301701786933",
                },
                "description": "Ingests order data",
                "end_time": 1629989930171,
                "execution_duration": 0,
                "job_cluster_key": "auto_scaling_cluster",
                "libraries": [{"jar": "dbfs:/mnt/databricks/OrderIngest.jar"}],
                "run_id": 2112892,
                "run_page_url": (
                    "https://my-workspace.cloud.databricks.com/#job/39832/run/20"
                ),
                "setup_duration": 0,
                "spark_jar_task": {"main_class_name": "com.databricks.OrdersIngest"},
                "start_time": 1629989929660,
                "state": {
                    "life_cycle_state": "INTERNAL_ERROR",
                    "result_state": "FAILED",
                    "state_message": (
                        "Library installation failed for library due to user error."
                        " Error messages:\n'Manage' permissions are required to install"
                        " libraries on a cluster"
                    ),
                    "user_cancelled_or_timedout": False,
                },
                "task_key": "Orders_Ingest",
            },
            {
                "attempt_number": 0,
                "cleanup_duration": 0,
                "cluster_instance": {"cluster_id": "0923-164208-meows279"},
                "depends_on": [
                    {"task_key": "Orders_Ingest"},
                    {"task_key": "Sessionize"},
                ],
                "description": "Matches orders with user sessions",
                "end_time": 1629989930238,
                "execution_duration": 0,
                "new_cluster": {
                    "autoscale": {"max_workers": 16, "min_workers": 2},
                    "aws_attributes": {"availability": "SPOT", "zone_id": "us-west-2a"},
                    "node_type_id": "i3.xlarge",
                    "spark_conf": {"spark.speculation": True},
                    "spark_version": "7.3.x-scala2.12",
                },
                "notebook_task": {
                    "notebook_path": "/Users/user.name@databricks.com/Match",
                    "source": "WORKSPACE",
                },
                "run_id": 2112897,
                "run_page_url": (
                    "https://my-workspace.cloud.databricks.com/#job/39832/run/21"
                ),
                "setup_duration": 0,
                "start_time": 0,
                "state": {
                    "life_cycle_state": "SKIPPED",
                    "state_message": "An upstream task failed.",
                    "user_cancelled_or_timedout": False,
                },
                "task_key": "Match",
            },
            {
                "attempt_number": 0,
                "cleanup_duration": 0,
                "cluster_instance": {
                    "cluster_id": "0923-164208-meows279",
                    "spark_context_id": "4348585301701786933",
                },
                "description": "Extracts session data from events",
                "end_time": 1629989930144,
                "execution_duration": 0,
                "existing_cluster_id": "0923-164208-meows279",
                "libraries": [{"jar": "dbfs:/mnt/databricks/Sessionize.jar"}],
                "run_id": 2112902,
                "run_page_url": (
                    "https://my-workspace.cloud.databricks.com/#job/39832/run/22"
                ),
                "setup_duration": 0,
                "spark_jar_task": {"main_class_name": "com.databricks.Sessionize"},
                "start_time": 1629989929668,
                "state": {
                    "life_cycle_state": "INTERNAL_ERROR",
                    "result_state": "FAILED",
                    "state_message": (
                        "Library installation failed for library due to user error."
                        " Error messages:\n'Manage' permissions are required to install"
                        " libraries on a cluster"
                    ),
                    "user_cancelled_or_timedout": False,
                },
                "task_key": "Sessionize",
            },
        ],
        max_items=100,
    )
    trigger: Optional[TriggerType] = Field(
        None, description="The type of trigger that fired this run."
    )

RunLifeCycleState

Bases: str, Enum

* `PENDING`: The run has been triggered. If there is not already an active run of the same job, the cluster and execution context are being prepared. If there is already an active run of the same job, the run immediately transitions into the `SKIPPED` state without preparing any resources.

• RUNNING: The task of this run is being executed.
• TERMINATING: The task of this run has completed, and the cluster and execution context are being cleaned up.
• TERMINATED: The task of this run has completed, and the cluster and execution context have been cleaned up. This state is terminal.
• SKIPPED: This run was aborted because a previous run of the same job was already active. This state is terminal.
• INTERNAL_ERROR: An exceptional state that indicates a failure in the Jobs service, such as network failure over a long period. If a run on a new cluster ends in the INTERNAL_ERROR state, the Jobs service terminates the cluster as soon as possible. This state is terminal.
• BLOCKED: The run is blocked on an upstream dependency.
• WAITING_FOR_RETRY: The run is waiting for a retry.

Source code in prefect_databricks/models/jobs.py

class RunLifeCycleState(str, Enum):
    """
        * `PENDING`: The run has been triggered. If there is not already an active run of the same job, the cluster and execution context are being prepared. If there is already an active run of the same job, the run immediately transitions into the `SKIPPED` state without preparing any resources.
    * `RUNNING`: The task of this run is being executed.
    * `TERMINATING`: The task of this run has completed, and the cluster and execution context are being cleaned up.
    * `TERMINATED`: The task of this run has completed, and the cluster and execution context have been cleaned up. This state is terminal.
    * `SKIPPED`: This run was aborted because a previous run of the same job was already active. This state is terminal.
    * `INTERNAL_ERROR`: An exceptional state that indicates a failure in the Jobs service, such as network failure over a long period. If a run on a new cluster ends in the `INTERNAL_ERROR` state, the Jobs service terminates the cluster as soon as possible. This state is terminal.
    * `BLOCKED`: The run is blocked on an upstream dependency.
    * `WAITING_FOR_RETRY`: The run is waiting for a retry.
    """

    terminated = "TERMINATED"
    pending = "PENDING"
    running = "RUNNING"
    terminating = "TERMINATING"
    skipped = "SKIPPED"
    internalerror = "INTERNAL_ERROR"
    blocked = "BLOCKED"
    waitingforretry = "WAITING_FOR_RETRY"

RunNowInput

Bases: BaseModel

See source code for the fields' description.

Source code in prefect_databricks/models/jobs.py

class RunNowInput(BaseModel):
    """
    See source code for the fields' description.
    """

    class Config:
        extra = Extra.allow
        allow_mutation = False

    idempotency_token: Optional[str] = Field(
        None,
        description=(
            "An optional token to guarantee the idempotency of job run requests. If a"
            " run with the provided token already exists, the request does not create a"
            " new run but returns the ID of the existing run instead. If a run with the"
            " provided token is deleted, an error is returned.\n\nIf you specify the"
            " idempotency token, upon failure you can retry until the request succeeds."
            " Databricks guarantees that exactly one run is launched with that"
            " idempotency token.\n\nThis token must have at most 64 characters.\n\nFor"
            " more information, see [How to ensure idempotency for"
            " jobs](https://kb.databricks.com/jobs/jobs-idempotency.html)."
        ),
        example="8f018174-4792-40d5-bcbc-3e6a527352c8",
    )
    job_id: Optional[int] = Field(
        None, description="The ID of the job to be executed", example=11223344
    )

RunParameters

Bases: BaseModel

See source code for the fields' description.

Source code in prefect_databricks/models/jobs.py

class RunParameters(BaseModel):
    """
    See source code for the fields' description.
    """

    class Config:
        extra = Extra.allow
        allow_mutation = False

    dbt_commands: Optional[List] = Field(
        None,
        description=(
            "An array of commands to execute for jobs with the dbt task, for example"
            ' `"dbt_commands": ["dbt deps", "dbt seed", "dbt run"]`'
        ),
        example=["dbt deps", "dbt seed", "dbt run"],
    )
    jar_params: Optional[List[str]] = Field(
        None,
        description=(
            "A list of parameters for jobs with Spark JAR tasks, for example"
            ' `"jar_params": ["john doe", "35"]`. The parameters are used to invoke the'
            " main function of the main class specified in the Spark JAR task. If not"
            " specified upon `run-now`, it defaults to an empty list. jar_params cannot"
            " be specified in conjunction with notebook_params. The JSON representation"
            ' of this field (for example `{"jar_params":["john doe","35"]}`) cannot'
            " exceed 10,000 bytes.\n\nUse [Task parameter"
            " variables](https://docs.databricks.com/jobs.html#parameter-variables) to"
            " set parameters containing information about job runs."
        ),
        example=["john", "doe", "35"],
    )
    notebook_params: Optional[Dict[str, Any]] = Field(
        None,
        description=(
            "A map from keys to values for jobs with notebook task, for example"
            ' `"notebook_params": {"name": "john doe", "age": "35"}`. The map is passed'
            " to the notebook and is accessible through the"
            " [dbutils.widgets.get](https://docs.databricks.com/dev-tools/databricks-utils.html#dbutils-widgets)"
            " function.\n\nIf not specified upon `run-now`, the triggered run uses the"
            " job’s base parameters.\n\nnotebook_params cannot be specified in"
            " conjunction with jar_params.\n\nUse [Task parameter"
            " variables](https://docs.databricks.com/jobs.html#parameter-variables) to"
            " set parameters containing information about job runs.\n\nThe JSON"
            " representation of this field (for example"
            ' `{"notebook_params":{"name":"john doe","age":"35"}}`) cannot exceed'
            " 10,000 bytes."
        ),
        example={"age": "35", "name": "john doe"},
    )
    pipeline_params: Optional[PipelineParams] = None
    python_named_params: Optional[Dict[str, Any]] = Field(
        None,
        description=(
            "A map from keys to values for jobs with Python wheel task, for example"
            ' `"python_named_params": {"name": "task", "data":'
            ' "dbfs:/path/to/data.json"}`.'
        ),
        example={"data": "dbfs:/path/to/data.json", "name": "task"},
    )
    python_params: Optional[List[str]] = Field(
        None,
        description=(
            "A list of parameters for jobs with Python tasks, for example"
            ' `"python_params": ["john doe", "35"]`. The parameters are passed to'
            " Python file as command-line parameters. If specified upon `run-now`, it"
            " would overwrite the parameters specified in job setting. The JSON"
            ' representation of this field (for example `{"python_params":["john'
            ' doe","35"]}`) cannot exceed 10,000 bytes.\n\nUse [Task parameter'
            " variables](https://docs.databricks.com/jobs.html#parameter-variables) to"
            " set parameters containing information about job"
            " runs.\n\nImportant\n\nThese parameters accept only Latin characters"
            " (ASCII character set). Using non-ASCII characters returns an error."
            " Examples of invalid, non-ASCII characters are Chinese, Japanese kanjis,"
            " and emojis."
        ),
        example=["john doe", "35"],
    )
    spark_submit_params: Optional[List[str]] = Field(
        None,
        description=(
            "A list of parameters for jobs with spark submit task, for example"
            ' `"spark_submit_params": ["--class",'
            ' "org.apache.spark.examples.SparkPi"]`. The parameters are passed to'
            " spark-submit script as command-line parameters. If specified upon"
            " `run-now`, it would overwrite the parameters specified in job setting."
            " The JSON representation of this field (for example"
            ' `{"python_params":["john doe","35"]}`) cannot exceed 10,000 bytes.\n\nUse'
            " [Task parameter"
            " variables](https://docs.databricks.com/jobs.html#parameter-variables) to"
            " set parameters containing information about job"
            " runs.\n\nImportant\n\nThese parameters accept only Latin characters"
            " (ASCII character set). Using non-ASCII characters returns an error."
            " Examples of invalid, non-ASCII characters are Chinese, Japanese kanjis,"
            " and emojis."
        ),
        example=["--class", "org.apache.spark.examples.SparkPi"],
    )
    sql_params: Optional[Dict[str, Any]] = Field(
        None,
        description=(
            'A map from keys to values for SQL tasks, for example `"sql_params":'
            ' {"name": "john doe", "age": "35"}`. The SQL alert task does not support'
            " custom parameters."
        ),
        example={"age": "35", "name": "john doe"},
    )

RunResultState

Bases: str, Enum

* `SUCCESS`: The task completed successfully.

• FAILED: The task completed with an error.
• TIMEDOUT: The run was stopped after reaching the timeout.
• CANCELED: The run was canceled at user request.

Source code in prefect_databricks/models/jobs.py

class RunResultState(str, Enum):
    """
        * `SUCCESS`: The task completed successfully.
    * `FAILED`: The task completed with an error.
    * `TIMEDOUT`: The run was stopped after reaching the timeout.
    * `CANCELED`: The run was canceled at user request.
    """

    success = "SUCCESS"
    failed = "FAILED"
    timedout = "TIMEDOUT"
    canceled = "CANCELED"

RunState

Bases: BaseModel

See source code for the fields' description.

The result and lifecycle state of the run.

Source code in prefect_databricks/models/jobs.py

class RunState(BaseModel):
    """
    See source code for the fields' description.

    The result and lifecycle state of the run.
    """

    class Config:
        extra = Extra.allow
        allow_mutation = False

    life_cycle_state: Optional[RunLifeCycleState] = Field(
        None,
        description=(
            "A description of a run’s current location in the run lifecycle. This field"
            " is always available in the response."
        ),
    )
    result_state: Optional[RunResultState] = None
    state_message: Optional[str] = Field(
        None,
        description=(
            "A descriptive message for the current state. This field is unstructured,"
            " and its exact format is subject to change."
        ),
        example="",
    )
    user_cancelled_or_timedout: Optional[bool] = Field(
        None,
        description=(
            "Whether a run was canceled manually by a user or by the scheduler because"
            " the run timed out."
        ),
        example=False,
    )

RunSubmitSettings

Bases: BaseModel

See source code for the fields' description.

Source code in prefect_databricks/models/jobs.py

class RunSubmitSettings(BaseModel):
    """
    See source code for the fields' description.
    """

    class Config:
        extra = Extra.allow
        allow_mutation = False

    git_source: Optional[GitSource1] = Field(
        None,
        description=(
            "This functionality is in Public Preview.\n\nAn optional specification for"
            " a remote repository containing the notebooks used by this job's notebook"
            " tasks."
        ),
        example={
            "git_branch": "main",
            "git_provider": "gitHub",
            "git_url": "https://github.com/databricks/databricks-cli",
        },
    )
    idempotency_token: Optional[str] = Field(
        None,
        description=(
            "An optional token that can be used to guarantee the idempotency of job run"
            " requests. If a run with the provided token already exists, the request"
            " does not create a new run but returns the ID of the existing run instead."
            " If a run with the provided token is deleted, an error is returned.\n\nIf"
            " you specify the idempotency token, upon failure you can retry until the"
            " request succeeds. Databricks guarantees that exactly one run is launched"
            " with that idempotency token.\n\nThis token must have at most 64"
            " characters.\n\nFor more information, see [How to ensure idempotency for"
            " jobs](https://kb.databricks.com/jobs/jobs-idempotency.html)."
        ),
        example="8f018174-4792-40d5-bcbc-3e6a527352c8",
    )
    run_name: Optional[str] = Field(
        None,
        description="An optional name for the run. The default value is `Untitled`.",
        example="A multitask job run",
    )
    tasks: Optional[List[RunSubmitTaskSettings]] = Field(
        None,
        example=[
            {
                "depends_on": [],
                "description": "Extracts session data from events",
                "existing_cluster_id": "0923-164208-meows279",
                "libraries": [{"jar": "dbfs:/mnt/databricks/Sessionize.jar"}],
                "spark_jar_task": {
                    "main_class_name": "com.databricks.Sessionize",
                    "parameters": ["--data", "dbfs:/path/to/data.json"],
                },
                "task_key": "Sessionize",
                "timeout_seconds": 86400,
            },
            {
                "depends_on": [],
                "description": "Ingests order data",
                "existing_cluster_id": "0923-164208-meows279",
                "libraries": [{"jar": "dbfs:/mnt/databricks/OrderIngest.jar"}],
                "spark_jar_task": {
                    "main_class_name": "com.databricks.OrdersIngest",
                    "parameters": ["--data", "dbfs:/path/to/order-data.json"],
                },
                "task_key": "Orders_Ingest",
                "timeout_seconds": 86400,
            },
            {
                "depends_on": [
                    {"task_key": "Orders_Ingest"},
                    {"task_key": "Sessionize"},
                ],
                "description": "Matches orders with user sessions",
                "new_cluster": {
                    "autoscale": {"max_workers": 16, "min_workers": 2},
                    "aws_attributes": {"availability": "SPOT", "zone_id": "us-west-2a"},
                    "node_type_id": "i3.xlarge",
                    "spark_conf": {"spark.speculation": True},
                    "spark_version": "7.3.x-scala2.12",
                },
                "notebook_task": {
                    "base_parameters": {"age": "35", "name": "John Doe"},
                    "notebook_path": "/Users/user.name@databricks.com/Match",
                    "source": "WORKSPACE",
                },
                "task_key": "Match",
                "timeout_seconds": 86400,
            },
        ],
        max_items=100,
    )
    timeout_seconds: Optional[int] = Field(
        None,
        description=(
            "An optional timeout applied to each run of this job. The default behavior"
            " is to have no timeout."
        ),
        example=86400,
    )
    webhook_notifications: Optional[WebhookNotifications] = Field(
        None,
        description=(
            "A collection of system notification IDs to notify when runs of this job"
            " begin or complete. The default behavior is to not send any system"
            " notifications."
        ),
    )

RunSubmitTaskSettings

Bases: BaseModel

See source code for the fields' description.

Source code in prefect_databricks/models/jobs.py

class RunSubmitTaskSettings(BaseModel):
    """
    See source code for the fields' description.
    """

    class Config:
        extra = Extra.allow
        allow_mutation = False

    dbt_task: Optional[DbtTask] = Field(
        None,
        description=(
            "If dbt_task, indicates that this must execute a dbt task. It requires both"
            " Databricks SQL and the ability to use a serverless or a pro SQL"
            " warehouse."
        ),
    )
    depends_on: Optional[TaskDependencies] = None
    existing_cluster_id: Optional[str] = Field(
        None,
        description=(
            "If existing_cluster_id, the ID of an existing cluster that is used for all"
            " runs of this task. When running tasks on an existing cluster, you may"
            " need to manually restart the cluster if it stops responding. We suggest"
            " running jobs on new clusters for greater reliability."
        ),
        example="0923-164208-meows279",
    )
    libraries: Optional[List[Library]] = Field(
        None,
        description=(
            "An optional list of libraries to be installed on the cluster that executes"
            " the task. The default value is an empty list."
        ),
    )
    new_cluster: Optional[NewCluster] = Field(
        None,
        description=(
            "If new_cluster, a description of a cluster that is created for each run."
        ),
    )
    notebook_task: Optional[NotebookTask] = Field(
        None,
        description=(
            "If notebook_task, indicates that this task must run a notebook. This field"
            " may not be specified in conjunction with spark_jar_task."
        ),
    )
    pipeline_task: Optional[PipelineTask] = Field(
        None,
        description=(
            "If pipeline_task, indicates that this task must execute a Pipeline."
        ),
    )
    python_wheel_task: Optional[PythonWheelTask] = Field(
        None,
        description=(
            "If python_wheel_task, indicates that this job must execute a PythonWheel."
        ),
    )
    spark_jar_task: Optional[SparkJarTask] = Field(
        None, description="If spark_jar_task, indicates that this task must run a JAR."
    )
    spark_python_task: Optional[SparkPythonTask] = Field(
        None,
        description=(
            "If spark_python_task, indicates that this task must run a Python file."
        ),
    )
    spark_submit_task: Optional[SparkSubmitTask] = Field(
        None,
        description=(
            "If spark_submit_task, indicates that this task must be launched by the"
            " spark submit script."
        ),
    )
    sql_task: Optional[SqlTask] = Field(
        None,
        description=(
            "If sql_task, indicates that this job must execute a SQL task. It requires"
            " both Databricks SQL and a serverless or a pro SQL warehouse."
        ),
    )
    task_key: TaskKey
    timeout_seconds: Optional[int] = Field(
        None,
        description=(
            "An optional timeout applied to each run of this job task. The default"
            " behavior is to have no timeout."
        ),
        example=86400,
    )

RunTask

Bases: BaseModel

See source code for the fields' description.

Source code in prefect_databricks/models/jobs.py

class RunTask(BaseModel):
    """
    See source code for the fields' description.
    """

    class Config:
        extra = Extra.allow
        allow_mutation = False

    attempt_number: Optional[int] = Field(
        None,
        description=(
            "The sequence number of this run attempt for a triggered job run. The"
            " initial attempt of a run has an attempt_number of 0\\. If the initial run"
            " attempt fails, and the job has a retry policy (`max_retries` \\> 0),"
            " subsequent runs are created with an `original_attempt_run_id` of the"
            " original attempt’s ID and an incrementing `attempt_number`. Runs are"
            " retried only until they succeed, and the maximum `attempt_number` is the"
            " same as the `max_retries` value for the job."
        ),
        example=0,
    )
    cleanup_duration: Optional[int] = Field(
        None,
        description=(
            "The time in milliseconds it took to terminate the cluster and clean up any"
            " associated artifacts. The total duration of the run is the sum of the"
            " setup_duration, the execution_duration, and the cleanup_duration."
        ),
        example=0,
    )
    cluster_instance: Optional[ClusterInstance] = Field(
        None,
        description=(
            "The cluster used for this run. If the run is specified to use a new"
            " cluster, this field is set once the Jobs service has requested a cluster"
            " for the run."
        ),
    )
    dbt_task: Optional[DbtTask] = Field(
        None,
        description=(
            "If dbt_task, indicates that this must execute a dbt task. It requires both"
            " Databricks SQL and the ability to use a serverless or a pro SQL"
            " warehouse."
        ),
    )
    depends_on: Optional[TaskDependencies] = None
    description: Optional[TaskDescription] = None
    end_time: Optional[int] = Field(
        None,
        description=(
            "The time at which this run ended in epoch milliseconds (milliseconds since"
            " 1/1/1970 UTC). This field is set to 0 if the job is still running."
        ),
        example=1625060863413,
    )
    execution_duration: Optional[int] = Field(
        None,
        description=(
            "The time in milliseconds it took to execute the commands in the JAR or"
            " notebook until they completed, failed, timed out, were cancelled, or"
            " encountered an unexpected error."
        ),
        example=0,
    )
    existing_cluster_id: Optional[str] = Field(
        None,
        description=(
            "If existing_cluster_id, the ID of an existing cluster that is used for all"
            " runs of this job. When running jobs on an existing cluster, you may need"
            " to manually restart the cluster if it stops responding. We suggest"
            " running jobs on new clusters for greater reliability."
        ),
    )
    git_source: Optional[GitSource1] = Field(
        None,
        description=(
            "This functionality is in Public Preview.\n\nAn optional specification for"
            " a remote repository containing the notebooks used by this job's notebook"
            " tasks."
        ),
        example={
            "git_branch": "main",
            "git_provider": "gitHub",
            "git_url": "https://github.com/databricks/databricks-cli",
        },
    )
    libraries: Optional[List[Library]] = Field(
        None,
        description=(
            "An optional list of libraries to be installed on the cluster that executes"
            " the job. The default value is an empty list."
        ),
    )
    new_cluster: Optional[NewCluster] = Field(
        None,
        description=(
            "If new_cluster, a description of a cluster that is created for each run."
        ),
    )
    notebook_task: Optional[NotebookTask] = Field(
        None,
        description=(
            "If notebook_task, indicates that this job must run a notebook. This field"
            " may not be specified in conjunction with spark_jar_task."
        ),
    )
    pipeline_task: Optional[PipelineTask] = Field(
        None,
        description=(
            "If pipeline_task, indicates that this job must execute a Pipeline."
        ),
    )
    python_wheel_task: Optional[PythonWheelTask] = Field(
        None,
        description=(
            "If python_wheel_task, indicates that this job must execute a PythonWheel."
        ),
    )
    run_id: Optional[int] = Field(
        None, description="The ID of the task run.", example=99887766
    )
    setup_duration: Optional[int] = Field(
        None,
        description=(
            "The time it took to set up the cluster in milliseconds. For runs that run"
            " on new clusters this is the cluster creation time, for runs that run on"
            " existing clusters this time should be very short."
        ),
        example=0,
    )
    spark_jar_task: Optional[SparkJarTask] = Field(
        None, description="If spark_jar_task, indicates that this job must run a JAR."
    )
    spark_python_task: Optional[SparkPythonTask] = Field(
        None,
        description=(
            "If spark_python_task, indicates that this job must run a Python file."
        ),
    )
    spark_submit_task: Optional[SparkSubmitTask] = Field(
        None,
        description=(
            "If spark_submit_task, indicates that this job must be launched by the"
            " spark submit script."
        ),
    )
    sql_task: Optional[SqlTask] = Field(
        None,
        description=(
            "If sql_task, indicates that this job must execute a SQL task. It requires"
            " both Databricks SQL and a serverless or a pro SQL warehouse."
        ),
    )
    start_time: Optional[int] = Field(
        None,
        description=(
            "The time at which this run was started in epoch milliseconds (milliseconds"
            " since 1/1/1970 UTC). This may not be the time when the job task starts"
            " executing, for example, if the job is scheduled to run on a new cluster,"
            " this is the time the cluster creation call is issued."
        ),
        example=1625060460483,
    )
    state: Optional[RunState] = Field(
        None, description="The result and lifecycle states of the run."
    )
    task_key: Optional[TaskKey] = None

RunType

Bases: str, Enum

The type of the run.

• JOB_RUN - Normal job run. A run created with Run now.
• WORKFLOW_RUN - Workflow run. A run created with dbutils.notebook.run.
• SUBMIT_RUN - Submit run. A run created with Run Submit.

Source code in prefect_databricks/models/jobs.py

class RunType(str, Enum):
    """
        The type of the run.
    * `JOB_RUN` \- Normal job run. A run created with [Run now](https://docs.databricks.com/dev-tools/api/latest/jobs.html#operation/JobsRunNow).
    * `WORKFLOW_RUN` \- Workflow run. A run created with [dbutils.notebook.run](https://docs.databricks.com/dev-tools/databricks-utils.html#dbutils-workflow).
    * `SUBMIT_RUN` \- Submit run. A run created with [Run Submit](https://docs.databricks.com/dev-tools/api/latest/jobs.html#operation/JobsRunsSubmit).
    """

    jobrun = "JOB_RUN"
    workflowrun = "WORKFLOW_RUN"
    submitrun = "SUBMIT_RUN"

RuntimeEngine

Bases: str, Enum

Decides which runtime engine to be use, e.g. Standard vs. Photon. If unspecified, the runtime engine is inferred from spark_version.

Source code in prefect_databricks/models/jobs.py

class RuntimeEngine(str, Enum):
    """
    Decides which runtime engine to be use, e.g. Standard vs. Photon. If unspecified, the runtime engine is inferred from spark_version.
    """

    standard = "STANDARD"
    photon = "PHOTON"

S3StorageInfo

Bases: BaseModel

See source code for the fields' description.

Source code in prefect_databricks/models/jobs.py

class S3StorageInfo(BaseModel):
    """
    See source code for the fields' description.
    """

    class Config:
        extra = Extra.allow
        allow_mutation = False

    canned_acl: Optional[str] = Field(
        None,
        description=(
            "(Optional) Set canned access control list. For example:"
            " `bucket-owner-full-control`. If canned_acl is set, the cluster instance"
            " profile must have `s3:PutObjectAcl` permission on the destination bucket"
            " and prefix. The full list of possible canned ACLs can be found at"
            " <https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl>."
            " By default only the object owner gets full control. If you are using"
            " cross account role for writing data, you may want to set"
            " `bucket-owner-full-control` to make bucket owner able to read the logs."
        ),
    )
    destination: Optional[str] = Field(
        None,
        description=(
            "S3 destination. For example: `s3://my-bucket/some-prefix` You must"
            " configure the cluster with an instance profile and the instance profile"
            " must have write access to the destination. You _cannot_ use AWS keys."
        ),
    )
    enable_encryption: Optional[bool] = Field(
        None, description="(Optional)Enable server side encryption, `false` by default."
    )
    encryption_type: Optional[str] = Field(
        None,
        description=(
            "(Optional) The encryption type, it could be `sse-s3` or `sse-kms`. It is"
            " used only when encryption is enabled and the default type is `sse-s3`."
        ),
    )
    endpoint: Optional[str] = Field(
        None,
        description=(
            "S3 endpoint. For example: `https://s3-us-west-2.amazonaws.com`. Either"
            " region or endpoint must be set. If both are set, endpoint is used."
        ),
    )
    kms_key: Optional[str] = Field(
        None,
        description=(
            "(Optional) KMS key used if encryption is enabled and encryption type is"
            " set to `sse-kms`."
        ),
    )
    region: Optional[str] = Field(
        None,
        description=(
            "S3 region. For example: `us-west-2`. Either region or endpoint must be"
            " set. If both are set, endpoint is used."
        ),
    )

ServicePrincipalName

Bases: BaseModel

See source code for the fields' description.

Source code in prefect_databricks/models/jobs.py

class ServicePrincipalName(BaseModel):
    """
    See source code for the fields' description.
    """

    class Config:
        extra = Extra.allow
        allow_mutation = False

    __root__: str = Field(
        ...,
        description="Name of an Azure service principal.",
        example="9f0621ee-b52b-11ea-b3de-0242ac130004",
    )

SparkConfPair

Bases: BaseModel

See source code for the fields' description.

An arbitrary object where the object key is a configuration propery name and the value is a configuration property value.

Source code in prefect_databricks/models/jobs.py

class SparkConfPair(BaseModel):
    """
    See source code for the fields' description.

    An arbitrary object where the object key is a configuration propery name and the value is a configuration property value.
    """

    class Config:
        extra = Extra.allow

        allow_mutation = False

SparkEnvPair

Bases: BaseModel

See source code for the fields' description.

An arbitrary object where the object key is an environment variable name and the value is an environment variable value.

Source code in prefect_databricks/models/jobs.py

class SparkEnvPair(BaseModel):
    """
    See source code for the fields' description.

    An arbitrary object where the object key is an environment variable name and the value is an environment variable value.
    """

    class Config:
        extra = Extra.allow

        allow_mutation = False

SparkJarTask

Bases: BaseModel

See source code for the fields' description.

Source code in prefect_databricks/models/jobs.py

class SparkJarTask(BaseModel):
    """
    See source code for the fields' description.
    """

    class Config:
        extra = Extra.allow
        allow_mutation = False

    jar_uri: Optional[str] = Field(
        None,
        deprecated=True,
        description=(
            "Deprecated since 04/2016\\. Provide a `jar` through the `libraries` field"
            " instead. For an example, see"
            " [Create](https://docs.databricks.com/dev-tools/api/latest/jobs.html#operation/JobsCreate)."
        ),
    )
    main_class_name: Optional[str] = Field(
        None,
        description=(
            "The full name of the class containing the main method to be executed. This"
            " class must be contained in a JAR provided as a library.\n\nThe code must"
            " use `SparkContext.getOrCreate` to obtain a Spark context; otherwise, runs"
            " of the job fail."
        ),
        example="com.databricks.ComputeModels",
    )
    parameters: Optional[List[str]] = Field(
        None,
        description=(
            "Parameters passed to the main method.\n\nUse [Task parameter"
            " variables](https://docs.databricks.com/jobs.html#parameter-variables) to"
            " set parameters containing information about job runs."
        ),
        example=["--data", "dbfs:/path/to/data.json"],
    )

SparkNode

Bases: BaseModel

See source code for the fields' description.

Source code in prefect_databricks/models/jobs.py

class SparkNode(BaseModel):
    """
    See source code for the fields' description.
    """

    class Config:
        extra = Extra.allow
        allow_mutation = False

    host_private_ip: Optional[str] = Field(
        None, description="The private IP address of the host instance."
    )
    instance_id: Optional[str] = Field(
        None,
        description=(
            "Globally unique identifier for the host instance from the cloud provider."
        ),
    )
    node_aws_attributes: Optional[SparkNodeAwsAttributes] = Field(
        None, description="Attributes specific to AWS for a Spark node."
    )
    node_id: Optional[str] = Field(
        None, description="Globally unique identifier for this node."
    )
    private_ip: Optional[str] = Field(
        None,
        description=(
            "Private IP address (typically a 10.x.x.x address) of the Spark node. This"
            " is different from the private IP address of the host instance."
        ),
    )
    public_dns: Optional[str] = Field(
        None,
        description=(
            "Public DNS address of this node. This address can be used to access the"
            " Spark JDBC server on the driver node. To communicate with the JDBC"
            " server, traffic must be manually authorized by adding security group"
            " rules to the “worker-unmanaged” security group via the AWS console."
        ),
    )
    start_timestamp: Optional[int] = Field(
        None,
        description="The timestamp (in millisecond) when the Spark node is launched.",
    )

SparkNodeAwsAttributes

Bases: BaseModel

See source code for the fields' description.

Source code in prefect_databricks/models/jobs.py

class SparkNodeAwsAttributes(BaseModel):
    """
    See source code for the fields' description.
    """

    class Config:
        extra = Extra.allow
        allow_mutation = False

    is_spot: Optional[bool] = Field(
        None, description="Whether this node is on an Amazon spot instance."
    )

SparkPythonTask

Bases: BaseModel

See source code for the fields' description.

Source code in prefect_databricks/models/jobs.py

class SparkPythonTask(BaseModel):
    """
    See source code for the fields' description.
    """

    class Config:
        extra = Extra.allow
        allow_mutation = False

    parameters: Optional[List[str]] = Field(
        None,
        description=(
            "Command line parameters passed to the Python file.\n\nUse [Task parameter"
            " variables](https://docs.databricks.com/jobs.html#parameter-variables) to"
            " set parameters containing information about job runs."
        ),
        example=["--data", "dbfs:/path/to/data.json"],
    )
    python_file: str = Field(
        ...,
        description=(
            "The Python file to be executed. Cloud file URIs (such as dbfs:/, s3:/,"
            " adls:/, gcs:/) and workspace paths are supported. For python files stored"
            " in the Databricks workspace, the path must be absolute and begin with"
            " `/Repos`. This field is required."
        ),
        example="dbfs:/path/to/file.py",
    )

SparkSubmitTask

Bases: BaseModel

See source code for the fields' description.

Source code in prefect_databricks/models/jobs.py

class SparkSubmitTask(BaseModel):
    """
    See source code for the fields' description.
    """

    class Config:
        extra = Extra.allow
        allow_mutation = False

    parameters: Optional[List[str]] = Field(
        None,
        description=(
            "Command-line parameters passed to spark submit.\n\nUse [Task parameter"
            " variables](https://docs.databricks.com/jobs.html#parameter-variables) to"
            " set parameters containing information about job runs."
        ),
        example=[
            "--class",
            "org.apache.spark.examples.SparkPi",
            "dbfs:/path/to/examples.jar",
            "10",
        ],
    )

SparkVersion

Bases: BaseModel

See source code for the fields' description.

Source code in prefect_databricks/models/jobs.py

class SparkVersion(BaseModel):
    """
    See source code for the fields' description.
    """

    class Config:
        extra = Extra.allow
        allow_mutation = False

    key: Optional[str] = Field(
        None,
        description=(
            "[Databricks Runtime"
            " version](https://docs.databricks.com/dev-tools/api/latest/index.html#programmatic-version)"
            " key, for example `7.3.x-scala2.12`. The value that must be provided as"
            " the `spark_version` when creating a new cluster. The exact runtime"
            " version may change over time for a “wildcard” version (that is,"
            " `7.3.x-scala2.12` is a “wildcard” version) with minor bug fixes."
        ),
    )
    name: Optional[str] = Field(
        None,
        description=(
            "A descriptive name for the runtime version, for example “Databricks"
            " Runtime 7.3 LTS”."
        ),
    )

SqlAlertOutput

Bases: BaseModel

See source code for the fields' description.

Source code in prefect_databricks/models/jobs.py

class SqlAlertOutput(BaseModel):
    """
    See source code for the fields' description.
    """

    class Config:
        extra = Extra.allow
        allow_mutation = False

    output_link: Optional[str] = Field(
        None, description="The link to find the output results."
    )
    query_text: Optional[str] = Field(
        None,
        description=(
            "The text of the SQL query. Can Run permission of the SQL query associated"
            " with the SQL alert is required to view this field."
        ),
    )
    sql_statements: Optional[SqlStatementOutput] = Field(
        None, description="Information about SQL statements executed in the run."
    )
    warehouse_id: Optional[str] = Field(
        None, description="The canonical identifier of the SQL warehouse."
    )

SqlDashboardOutput

Bases: BaseModel

See source code for the fields' description.

Source code in prefect_databricks/models/jobs.py

class SqlDashboardOutput(BaseModel):
    """
    See source code for the fields' description.
    """

    class Config:
        extra = Extra.allow
        allow_mutation = False

    widgets: Optional[SqlDashboardWidgetOutput] = Field(
        None,
        description=(
            "Widgets executed in the run. Only SQL query based widgets are listed."
        ),
    )

SqlDashboardWidgetOutput

Bases: BaseModel

See source code for the fields' description.

Source code in prefect_databricks/models/jobs.py

class SqlDashboardWidgetOutput(BaseModel):
    """
    See source code for the fields' description.
    """

    class Config:
        extra = Extra.allow
        allow_mutation = False

    end_time: Optional[int] = Field(
        None,
        description=(
            "Time (in epoch milliseconds) when execution of the SQL widget ends."
        ),
    )
    error: Optional[SqlOutputError] = Field(
        None, description="The information about the error when execution fails."
    )
    output_link: Optional[str] = Field(
        None, description="The link to find the output results."
    )
    start_time: Optional[int] = Field(
        None,
        description=(
            "Time (in epoch milliseconds) when execution of the SQL widget starts."
        ),
    )
    status: Optional[
        Literal["PENDING", "RUNNING", "SUCCESS", "FAILED", "CANCELLED"]
    ] = Field(None, description="The execution status of the SQL widget.")
    widget_id: Optional[str] = Field(
        None, description="The canonical identifier of the SQL widget."
    )
    widget_title: Optional[str] = Field(
        None, description="The title of the SQL widget."
    )

SqlOutput

Bases: BaseModel

See source code for the fields' description.

Source code in prefect_databricks/models/jobs.py

class SqlOutput(BaseModel):
    """
    See source code for the fields' description.
    """

    class Config:
        extra = Extra.allow
        allow_mutation = False

    alert_output: Optional[SqlAlertOutput] = Field(
        None, description="The output of a SQL alert task, if available."
    )
    dashboard_output: Optional[SqlDashboardOutput] = Field(
        None, description="The output of a SQL dashboard task, if available."
    )
    query_output: Optional[SqlQueryOutput] = Field(
        None, description="The output of a SQL query task, if available."
    )

SqlOutputError

Bases: BaseModel

See source code for the fields' description.

Source code in prefect_databricks/models/jobs.py

class SqlOutputError(BaseModel):
    """
    See source code for the fields' description.
    """

    class Config:
        extra = Extra.allow
        allow_mutation = False

    message: Optional[str] = Field(
        None, description="The error message when execution fails."
    )

SqlQueryOutput

Bases: BaseModel

See source code for the fields' description.

Source code in prefect_databricks/models/jobs.py

class SqlQueryOutput(BaseModel):
    """
    See source code for the fields' description.
    """

    class Config:
        extra = Extra.allow
        allow_mutation = False

    output_link: Optional[str] = Field(
        None, description="The link to find the output results."
    )
    query_text: Optional[str] = Field(
        None,
        description=(
            "The text of the SQL query. Can Run permission of the SQL query is required"
            " to view this field."
        ),
    )
    sql_statements: Optional[SqlStatementOutput] = Field(
        None, description="Information about SQL statements executed in the run."
    )
    warehouse_id: Optional[str] = Field(
        None, description="The canonical identifier of the SQL warehouse."
    )

SqlStatementOutput

Bases: BaseModel

See source code for the fields' description.

Source code in prefect_databricks/models/jobs.py

class SqlStatementOutput(BaseModel):
    """
    See source code for the fields' description.
    """

    class Config:
        extra = Extra.allow
        allow_mutation = False

    lookup_key: Optional[str] = Field(
        None, description="A key that can be used to look up query details."
    )

SqlTask

Bases: BaseModel

See source code for the fields' description.

Source code in prefect_databricks/models/jobs.py

class SqlTask(BaseModel):
    """
    See source code for the fields' description.
    """

    class Config:
        extra = Extra.allow
        allow_mutation = False

    alert: Optional[SqlTaskAlert] = Field(
        None, description="If alert, indicates that this job must refresh a SQL alert."
    )
    dashboard: Optional[SqlTaskDashboard] = Field(
        None,
        description=(
            "If dashboard, indicates that this job must refresh a SQL dashboard."
        ),
    )
    parameters: Optional[Dict[str, Any]] = Field(
        None,
        description=(
            "Parameters to be used for each run of this job. The SQL alert task does"
            " not support custom parameters."
        ),
        example={"age": 35, "name": "John Doe"},
    )
    query: Optional[SqlTaskQuery] = Field(
        None, description="If query, indicates that this job must execute a SQL query."
    )
    warehouse_id: str = Field(
        ...,
        description=(
            "The canonical identifier of the SQL warehouse. Only serverless and pro SQL"
            " warehouses are supported."
        ),
    )

SqlTaskAlert

Bases: BaseModel

See source code for the fields' description.

Source code in prefect_databricks/models/jobs.py

class SqlTaskAlert(BaseModel):
    """
    See source code for the fields' description.
    """

    class Config:
        extra = Extra.allow
        allow_mutation = False

    alert_id: str = Field(..., description="The canonical identifier of the SQL alert.")

SqlTaskDashboard

Bases: BaseModel

See source code for the fields' description.

Source code in prefect_databricks/models/jobs.py

class SqlTaskDashboard(BaseModel):
    """
    See source code for the fields' description.
    """

    class Config:
        extra = Extra.allow
        allow_mutation = False

    dashboard_id: str = Field(
        ..., description="The canonical identifier of the SQL dashboard."
    )

SqlTaskQuery

Bases: BaseModel

See source code for the fields' description.

Source code in prefect_databricks/models/jobs.py

class SqlTaskQuery(BaseModel):
    """
    See source code for the fields' description.
    """

    class Config:
        extra = Extra.allow
        allow_mutation = False

    query_id: str = Field(..., description="The canonical identifier of the SQL query.")

TaskDependencies

Bases: BaseModel

See source code for the fields' description.

An optional array of objects specifying the dependency graph of the task. All tasks specified in this field must complete successfully before executing this task.

The key is task_key, and the value is the name assigned to the dependent task. This field is required when a job consists of more than one task.

Source code in prefect_databricks/models/jobs.py

class TaskDependencies(BaseModel):
    """
    See source code for the fields' description.

        An optional array of objects specifying the dependency graph of the task. All tasks specified in this field must complete successfully before executing this task.
    The key is `task_key`, and the value is the name assigned to the dependent task.
    This field is required when a job consists of more than one task.
    """

    class Config:
        extra = Extra.allow
        allow_mutation = False

    __root__: List[TaskDependency] = Field(
        ...,
        description=(
            "An optional array of objects specifying the dependency graph of the task."
            " All tasks specified in this field must complete successfully before"
            " executing this task.\nThe key is `task_key`, and the value is the name"
            " assigned to the dependent task.\nThis field is required when a job"
            " consists of more than one task."
        ),
        example=[{"task_key": "Previous_Task_Key"}, {"task_key": "Other_Task_Key"}],
    )

TaskDependency

Bases: BaseModel

See source code for the fields' description.

Source code in prefect_databricks/models/jobs.py

class TaskDependency(BaseModel):
    """
    See source code for the fields' description.
    """

    class Config:
        extra = Extra.allow
        allow_mutation = False

    task_key: Optional[str] = None

TaskDescription

Bases: BaseModel

See source code for the fields' description.

Source code in prefect_databricks/models/jobs.py

class TaskDescription(BaseModel):
    """
    See source code for the fields' description.
    """

    class Config:
        extra = Extra.allow
        allow_mutation = False

    __root__: str = Field(
        ...,
        description=(
            "An optional description for this task.\nThe maximum length is 4096 bytes."
        ),
        example="This is the description for this task.",
        max_length=4096,
    )

TaskKey

Bases: BaseModel

See source code for the fields' description.

Source code in prefect_databricks/models/jobs.py

class TaskKey(BaseModel):
    """
    See source code for the fields' description.
    """

    class Config:
        extra = Extra.allow
        allow_mutation = False

    __root__: str = Field(
        ...,
        description=(
            "A unique name for the task. This field is used to refer to this task from"
            " other tasks.\nThis field is required and must be unique within its parent"
            " job.\nOn Update or Reset, this field is used to reference the tasks to be"
            " updated or reset.\nThe maximum length is 100 characters."
        ),
        example="Task_Key",
        max_length=100,
        min_length=1,
        regex="^[\\w\\-]+$",
    )

TerminationCode

Bases: str, Enum

* USER_REQUEST: A user terminated the cluster directly. Parameters should include a `username` field that indicates the specific user who terminated the cluster.

• JOB_FINISHED: The cluster was launched by a job, and terminated when the job completed.
• INACTIVITY: The cluster was terminated since it was idle.
• CLOUD_PROVIDER_SHUTDOWN: The instance that hosted the Spark driver was terminated by the cloud provider. In AWS, for example, AWS may retire instances and directly shut them down. Parameters should include an aws_instance_state_reason field indicating the AWS-provided reason why the instance was terminated.
• COMMUNICATION_LOST: Databricks lost connection to services on the driver instance. For example, this can happen when problems arise in cloud networking infrastructure, or when the instance itself becomes unhealthy.
• CLOUD_PROVIDER_LAUNCH_FAILURE: Databricks experienced a cloud provider failure when requesting instances to launch clusters. For example, AWS limits the number of running instances and EBS volumes. If you ask Databricks to launch a cluster that requires instances or EBS volumes that exceed your AWS limit, the cluster fails with this status code. Parameters should include one of aws_api_error_code, aws_instance_state_reason, or aws_spot_request_status to indicate the AWS-provided reason why Databricks could not request the required instances for the cluster.
• SPARK_STARTUP_FAILURE: The cluster failed to initialize. Possible reasons may include failure to create the environment for Spark or issues launching the Spark master and worker processes.
• INVALID_ARGUMENT: Cannot launch the cluster because the user specified an invalid argument. For example, the user might specify an invalid runtime version for the cluster.
• UNEXPECTED_LAUNCH_FAILURE: While launching this cluster, Databricks failed to complete critical setup steps, terminating the cluster.
• INTERNAL_ERROR: Databricks encountered an unexpected error that forced the running cluster to be terminated. Contact Databricks support for additional details.
• SPARK_ERROR: The Spark driver failed to start. Possible reasons may include incompatible libraries and initialization scripts that corrupted the Spark container.
• METASTORE_COMPONENT_UNHEALTHY: The cluster failed to start because the external metastore could not be reached. Refer to Troubleshooting.
• DBFS_COMPONENT_UNHEALTHY: The cluster failed to start because Databricks File System (DBFS) could not be reached.
• DRIVER_UNREACHABLE: Databricks was not able to access the Spark driver, because it was not reachable.
• DRIVER_UNRESPONSIVE: Databricks was not able to access the Spark driver, because it was unresponsive.
• INSTANCE_UNREACHABLE: Databricks was not able to access instances in order to start the cluster. This can be a transient networking issue. If the problem persists, this usually indicates a networking environment misconfiguration.
• CONTAINER_LAUNCH_FAILURE: Databricks was unable to launch containers on worker nodes for the cluster. Have your admin check your network configuration.
• INSTANCE_POOL_CLUSTER_FAILURE: Pool backed cluster specific failure. Refer to Pools for details.
• REQUEST_REJECTED: Databricks cannot handle the request at this moment. Try again later and contact Databricks if the problem persists.
• INIT_SCRIPT_FAILURE: Databricks cannot load and run a cluster-scoped init script on one of the cluster’s nodes, or the init script terminates with a non-zero exit code. Refer to Init script logs.
• TRIAL_EXPIRED: The Databricks trial subscription expired.

Source code in prefect_databricks/models/jobs.py

class TerminationCode(str, Enum):
    """
        * USER_REQUEST: A user terminated the cluster directly. Parameters should include a `username` field that indicates the specific user who terminated the cluster.
    * JOB_FINISHED: The cluster was launched by a job, and terminated when the job completed.
    * INACTIVITY: The cluster was terminated since it was idle.
    * CLOUD_PROVIDER_SHUTDOWN: The instance that hosted the Spark driver was terminated by the cloud provider. In AWS, for example, AWS may retire instances and directly shut them down. Parameters should include an `aws_instance_state_reason` field indicating the AWS-provided reason why the instance was terminated.
    * COMMUNICATION_LOST: Databricks lost connection to services on the driver instance. For example, this can happen when problems arise in cloud networking infrastructure, or when the instance itself becomes unhealthy.
    * CLOUD_PROVIDER_LAUNCH_FAILURE: Databricks experienced a cloud provider failure when requesting instances to launch clusters. For example, AWS limits the number of running instances and EBS volumes. If you ask Databricks to launch a cluster that requires instances or EBS volumes that exceed your AWS limit, the cluster fails with this status code. Parameters should include one of `aws_api_error_code`, `aws_instance_state_reason`, or `aws_spot_request_status` to indicate the AWS-provided reason why Databricks could not request the required instances for the cluster.
    * SPARK_STARTUP_FAILURE: The cluster failed to initialize. Possible reasons may include failure to create the environment for Spark or issues launching the Spark master and worker processes.
    * INVALID_ARGUMENT: Cannot launch the cluster because the user specified an invalid argument. For example, the user might specify an invalid runtime version for the cluster.
    * UNEXPECTED_LAUNCH_FAILURE: While launching this cluster, Databricks failed to complete critical setup steps, terminating the cluster.
    * INTERNAL_ERROR: Databricks encountered an unexpected error that forced the running cluster to be terminated. Contact Databricks support for additional details.
    * SPARK_ERROR: The Spark driver failed to start. Possible reasons may include incompatible libraries and initialization scripts that corrupted the Spark container.
    * METASTORE_COMPONENT_UNHEALTHY: The cluster failed to start because the external metastore could not be reached. Refer to [Troubleshooting](https://docs.databricks.com/data/metastores/external-hive-metastore.html#troubleshooting).
    * DBFS_COMPONENT_UNHEALTHY: The cluster failed to start because Databricks File System (DBFS) could not be reached.
    * DRIVER_UNREACHABLE: Databricks was not able to access the Spark driver, because it was not reachable.
    * DRIVER_UNRESPONSIVE: Databricks was not able to access the Spark driver, because it was unresponsive.
    * INSTANCE_UNREACHABLE: Databricks was not able to access instances in order to start the cluster. This can be a transient networking issue. If the problem persists, this usually indicates a networking environment misconfiguration.
    * CONTAINER_LAUNCH_FAILURE: Databricks was unable to launch containers on worker nodes for the cluster. Have your admin check your network configuration.
    * INSTANCE_POOL_CLUSTER_FAILURE: Pool backed cluster specific failure. Refer to [Pools](https://docs.databricks.com/clusters/instance-pools/index.html) for details.
    * REQUEST_REJECTED: Databricks cannot handle the request at this moment. Try again later and contact Databricks if the problem persists.
    * INIT_SCRIPT_FAILURE: Databricks cannot load and run a cluster-scoped init script on one of the cluster’s nodes, or the init script terminates with a non-zero exit code. Refer to [Init script logs](https://docs.databricks.com/clusters/init-scripts.html#init-script-log).
    * TRIAL_EXPIRED: The Databricks trial subscription expired.
    """

    userrequest = "USER_REQUEST"
    jobfinished = "JOB_FINISHED"
    inactivity = "INACTIVITY"
    cloudprovidershutdown = "CLOUD_PROVIDER_SHUTDOWN"
    communicationlost = "COMMUNICATION_LOST"
    cloudproviderlaunchfailure = "CLOUD_PROVIDER_LAUNCH_FAILURE"
    sparkstartupfailure = "SPARK_STARTUP_FAILURE"
    invalidargument = "INVALID_ARGUMENT"
    unexpectedlaunchfailure = "UNEXPECTED_LAUNCH_FAILURE"
    internalerror = "INTERNAL_ERROR"
    sparkerror = "SPARK_ERROR"
    metastorecomponentunhealthy = "METASTORE_COMPONENT_UNHEALTHY"
    dbfscomponentunhealthy = "DBFS_COMPONENT_UNHEALTHY"
    driverunreachable = "DRIVER_UNREACHABLE"
    driverunresponsive = "DRIVER_UNRESPONSIVE"
    instanceunreachable = "INSTANCE_UNREACHABLE"
    containerlaunchfailure = "CONTAINER_LAUNCH_FAILURE"
    instancepoolclusterfailure = "INSTANCE_POOL_CLUSTER_FAILURE"
    requestrejected = "REQUEST_REJECTED"
    initscriptfailure = "INIT_SCRIPT_FAILURE"
    trialexpired = "TRIAL_EXPIRED"

TerminationParameter

Bases: BaseModel

See source code for the fields' description.

Source code in prefect_databricks/models/jobs.py

class TerminationParameter(BaseModel):
    """
    See source code for the fields' description.
    """

    class Config:
        extra = Extra.allow
        allow_mutation = False

    aws_api_error_code: Optional[str] = Field(
        None,
        description=(
            "The AWS provided error code describing why cluster nodes could not be"
            " provisioned. For example, `InstanceLimitExceeded` indicates that the"
            " limit of EC2 instances for a specific instance type has been exceeded."
            " For reference, see:"
            " <https://docs.aws.amazon.com/AWSEC2/latest/APIReference/query-api-troubleshooting.html>."
        ),
    )
    aws_error_message: Optional[str] = Field(
        None,
        description=(
            "Human-readable context of various failures from AWS. This field is"
            " unstructured, and its exact format is subject to change."
        ),
    )
    aws_impaired_status_details: Optional[str] = Field(
        None,
        description=(
            "The AWS provided status check which failed and induced a node loss. This"
            " status may correspond to a failed instance or system check. For"
            " reference, see"
            " <https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/monitoring-system-instance-status-check.html>."
        ),
    )
    aws_instance_state_reason: Optional[str] = Field(
        None,
        description=(
            "The AWS provided state reason describing why the driver node was"
            " terminated. For example, `Client.VolumeLimitExceeded` indicates that the"
            " limit of EBS volumes or total EBS volume storage has been exceeded. For"
            " reference, see"
            " <https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_StateReason.html>."
        ),
    )
    aws_instance_status_event: Optional[str] = Field(
        None,
        description=(
            "The AWS provided scheduled event (for example reboot) which induced a node"
            " loss. For reference, see"
            " <https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/monitoring-instances-status-check_sched.html>."
        ),
    )
    aws_spot_request_fault_code: Optional[str] = Field(
        None,
        description=(
            "Provides additional details when a spot request fails. For example"
            " `InsufficientFreeAddressesInSubnet` indicates the subnet does not have"
            " free IP addresses to accommodate the new instance. For reference, see"
            " <https://docs.aws.amazon.com/cli/latest/reference/ec2/describe-spot-instance-requests.html>."
        ),
    )
    aws_spot_request_status: Optional[str] = Field(
        None,
        description=(
            "Describes why a spot request could not be fulfilled. For example,"
            " `price-too-low` indicates that the max price was lower than the current"
            " spot price. For reference, see:"
            " <https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/spot-bid-status.html#spot-instance-bid-status-understand>."
        ),
    )
    databricks_error_message: Optional[str] = Field(
        None,
        description=(
            "Additional context that may explain the reason for cluster termination."
            " This field is unstructured, and its exact format is subject to change."
        ),
    )
    inactivity_duration_min: Optional[str] = Field(
        None,
        description=(
            "An idle cluster was shut down after being inactive for this duration."
        ),
    )
    instance_id: Optional[str] = Field(
        None, description="The ID of the instance that was hosting the Spark driver."
    )
    instance_pool_error_code: Optional[str] = Field(
        None,
        description=(
            "The [error"
            " code](https://docs.databricks.com/dev-tools/api/latest/clusters.html#clusterterminationreasonpoolclusterterminationcode)"
            " for cluster failures specific to a pool."
        ),
    )
    instance_pool_id: Optional[str] = Field(
        None, description="The ID of the instance pool the cluster is using."
    )
    username: Optional[str] = Field(
        None, description="The username of the user who terminated the cluster."
    )

TerminationReason

Bases: BaseModel

See source code for the fields' description.

Source code in prefect_databricks/models/jobs.py

class TerminationReason(BaseModel):
    """
    See source code for the fields' description.
    """

    class Config:
        extra = Extra.allow
        allow_mutation = False

    code: Optional[TerminationCode] = Field(
        None, description="Status code indicating why a cluster was terminated."
    )
    parameters: Optional[ParameterPair] = Field(
        None,
        description=(
            "Object containing a set of parameters that provide information about why a"
            " cluster was terminated."
        ),
    )
    type: Optional[TerminationType] = Field(
        None, description="Reason indicating why a cluster was terminated."
    )

TerminationType

Bases: str, Enum

* SUCCESS: Termination succeeded.

• CLIENT_ERROR: Non-retriable. Client must fix parameters before reattempting the cluster creation.
• SERVICE_FAULT: Databricks service issue. Client can retry.
• CLOUD_FAILURECloud provider infrastructure issue. Client can retry after the underlying issue is resolved.

Source code in prefect_databricks/models/jobs.py

class TerminationType(str, Enum):
    """
        * SUCCESS: Termination succeeded.
    * CLIENT_ERROR: Non-retriable. Client must fix parameters before reattempting the cluster creation.
    * SERVICE_FAULT: Databricks service issue. Client can retry.
    * CLOUD_FAILURECloud provider infrastructure issue. Client can retry after the underlying issue is resolved.

    """

    success = "SUCCESS"
    clienterror = "CLIENT_ERROR"
    servicefault = "SERVICE_FAULT"
    cloudfailure = "CLOUD_FAILURE"

TriggerType

Bases: str, Enum

* `PERIODIC`: Schedules that periodically trigger runs, such as a cron scheduler.

• ONE_TIME: One time triggers that fire a single run. This occurs you triggered a single run on demand through the UI or the API.
• RETRY: Indicates a run that is triggered as a retry of a previously failed run. This occurs when you request to re-run the job in case of failures.

Source code in prefect_databricks/models/jobs.py

class TriggerType(str, Enum):
    """
        * `PERIODIC`: Schedules that periodically trigger runs, such as a cron scheduler.
    * `ONE_TIME`: One time triggers that fire a single run. This occurs you triggered a single run on demand through the UI or the API.
    * `RETRY`: Indicates a run that is triggered as a retry of a previously failed run. This occurs when you request to re-run the job in case of failures.
    """

    periodic = "PERIODIC"
    onetime = "ONE_TIME"
    retry = "RETRY"

UserName

Bases: BaseModel

See source code for the fields' description.

Source code in prefect_databricks/models/jobs.py

class UserName(BaseModel):
    """
    See source code for the fields' description.
    """

    class Config:
        extra = Extra.allow
        allow_mutation = False

    __root__: str = Field(
        ..., description="Email address for the user.", example="jsmith@example.com"
    )

ViewItem

Bases: BaseModel

See source code for the fields' description.

Source code in prefect_databricks/models/jobs.py

class ViewItem(BaseModel):
    """
    See source code for the fields' description.
    """

    class Config:
        extra = Extra.allow
        allow_mutation = False

    content: Optional[str] = Field(None, description="Content of the view.")
    name: Optional[str] = Field(
        None,
        description=(
            "Name of the view item. In the case of code view, it would be the"
            " notebook’s name. In the case of dashboard view, it would be the"
            " dashboard’s name."
        ),
    )
    type: Optional[ViewType] = Field(None, description="Type of the view item.")

ViewType

Bases: str, Enum

* `NOTEBOOK`: Notebook view item.

• DASHBOARD: Dashboard view item.

Source code in prefect_databricks/models/jobs.py

class ViewType(str, Enum):
    """
        * `NOTEBOOK`: Notebook view item.
    * `DASHBOARD`: Dashboard view item.
    """

    notebook = "NOTEBOOK"
    dashboard = "DASHBOARD"

ViewsToExport

Bases: str, Enum

* `CODE`: Code view of the notebook.

• DASHBOARDS: All dashboard views of the notebook.
• ALL: All views of the notebook.

Source code in prefect_databricks/models/jobs.py

class ViewsToExport(str, Enum):
    """
        * `CODE`: Code view of the notebook.
    * `DASHBOARDS`: All dashboard views of the notebook.
    * `ALL`: All views of the notebook.
    """

    code = "CODE"
    dashboards = "DASHBOARDS"
    all = "ALL"

WebhookNotifications

Bases: BaseModel

See source code for the fields' description.

Source code in prefect_databricks/models/jobs.py

class WebhookNotifications(BaseModel):
    """
    See source code for the fields' description.
    """

    class Config:
        extra = Extra.allow
        allow_mutation = False

    on_failure: Optional[List[OnFailureItem]] = Field(
        None,
        description=(
            "An optional list of notification IDs to call when the run fails. A maximum"
            " of 3 destinations can be specified for the `on_failure` property."
        ),
        example=[{"id": "0481e838-0a59-4eff-9541-a4ca6f149574"}],
    )
    on_start: Optional[List[OnStartItem]] = Field(
        None,
        description=(
            "An optional list of notification IDs to call when the run starts. A"
            " maximum of 3 destinations can be specified for the `on_start` property."
        ),
        example=[
            {"id": "03dd86e4-57ef-4818-a950-78e41a1d71ab"},
            {"id": "0481e838-0a59-4eff-9541-a4ca6f149574"},
        ],
    )
    on_success: Optional[List[OnSucces]] = Field(
        None,
        description=(
            "An optional list of notification IDs to call when the run completes"
            " successfully. A maximum of 3 destinations can be specified for the"
            " `on_success` property."
        ),
        example=[{"id": "03dd86e4-57ef-4818-a950-78e41a1d71ab"}],
    )