データフロー定義

この記事では、データフロー項目の定義構造の詳細について説明します。

定義パーツ

Metadata ContentDetails

ペイロードの内容について説明します

ComputeEngineSettings コンテンツ

QueryGroups の内容

QueriesMetadata コンテンツ

接続の内容

Metadata ContentDetails の例

{
  "formatVersion": "202502",
  "computeEngineSettings": {
    "allowFastCopy": true,
    "maxConcurrency": 1
  },
  "name": "SampleDataflowGen",
  "queryGroups": [
    
  ],
  "documentLocale": "en-US",
  "gatewayObjectId": null,
  "queriesMetadata": {
    "publicholidays": {
      "queryId": "a0a0a0a0-bbbb-cccc-dddd-e1e1e1e1e1e1",
      "queryName": "publicholidays",
      "queryGroupId": null,
      "isHidden": false,
      "loadEnabled": true
    },
    "MDF transform": {
      "queryId": "9c6d7c87-83db-4718-b382-c0c1aa1636d7",
      "queryName": "MDF transform",
      "loadEnabled": false
    }
  },
  "connections": [
    {
      "path": "Lakehouse",
      "kind": "Lakehouse",
      "connectionId": "{\"ClusterId\":\"b1b1b1b1-cccc-dddd-eeee-f2f2f2f2f2f2\",\"DatasourceId\":\"c2c2c2c2-dddd-eeee-ffff-a3a3a3a3a3a3\"}"
    }
  ],
  "fastCombine": false,
  "allowNativeQueries": true,
  "skipAutomaticTypeAndHeaderDetection": false
}

Mashup ContentDetails の例

[StagingDefinition = [Kind = "FastCopy"]]
section Section1;
shared publicholidays = 
let  Source = Lakehouse.Contents([]),  
#"Navigation 1" = Source{[workspaceId = "d3d3d3d3-eeee-ffff-aaaa-b4b4b4b4b4b4"]}[Data],  
#"Navigation 2" = #"Navigation 1"{[lakehouseId = "e4e4e4e4-ffff-aaaa-bbbb-c5c5c5c5c5c5"]}[Data],  
#"Navigation 3" = #"Navigation 2"{[Id = "publicholidays", ItemKind = "Table"]}[Data],  
#"Changed column type" = Table.TransformColumnTypes(#"Navigation 3", {{"normalizeHolidayName", type text}}),  
#"Lowercased text" = Table.TransformColumns(#"Changed column type", {{"countryRegionCode", each Text.Lower(_), type nullable text}}),  
#"Uppercased text" = Table.TransformColumns(#"Lowercased text", {{"normalizeHolidayName", each Text.Upper(_), type nullable text}}),  
#"Calculated text length" = Table.TransformColumns(#"Uppercased text", {{"countryOrRegion", each Text.Length(_), type nullable Int64.Type}})in  #"Calculated text length";
[ItemType = "MDF"]
shared #"MDF transform" = "___ExternalResource-Placeholder___";

MDFTransform ContentDetails

Mapping Data Flow ペイロードの内容について説明します。

MDFTransform プロパティの内容

MDFTransform TypeProperties コンテンツ

MDFTransform ソース コンテンツ

MDFTransform シンクの内容

MDFTransform 変換の内容

MDFTransform ExternalReferences コンテンツ

MDFTransform ContentDetails の例

{
  "name": "MDF",
  "properties": {
    "type": "MappingDataFlow",
    "typeProperties": {
      "sources": [
        {
          "name": "source1",
          "externalReferences": {
            "connection": "<source1ConnectionId>"
          }
        }
      ],
      "sinks": [
        {
          "name": "sink1",
          "externalReferences": {
            "connection": "<sink1ConnectionId>"
          }
        }
      ],
      "transformations": [
        {
          "name": "derivedColumn1"
        }
      ],
      "scriptLines": [
        "source(output(",
        "          Column1 as short,",
        "          Column2 as string,",
        "          Column3 as date,",
        "          Column4 as boolean,",
        "          Column5 as string,",
        "          Column6 as double",
        "     ),",
        "     useSchema: false,",
        "     allowSchemaDrift: true,",
        "     validateSchema: false,",
        "     ignoreNoFilesFound: false,",
        "     format: 'delimited',",
        "     fileSystem: 'folder1',",
        "     fileName: 'file1.csv',",
        "     columnDelimiter: ',',",
        "     escapeChar: '\\\\',",
        "     quoteChar: '\\\"',",
        "     columnNamesAsHeader: true) ~> source1",
        "source1 derive(isColumn6GreaterThan10 = Column6>10) ~> derivedColumn1",
        "derivedColumn1 sink(allowSchemaDrift: true,",
        "     validateSchema: false,",
        "     format: 'json',",
        "     fileSystem: 'output1',",
        "     umask: 0022,",
        "     preCommands: [],",
        "     postCommands: [],",
        "     skipDuplicateMapInputs: true,",
        "     skipDuplicateMapOutputs: true) ~> sink1"
      ]
    }
  }
}