Datacomponents

Datacomponents hold configuration for dataplugins, and are the building blocks for constructing pipelines. One dataplugin may be referenced by many datacomponents, each with a different set of properties for the dataplugin settings. One pipeline may reference multiple datacomponents.

Objects

Datacomponent

Path	Type	Format	Description
`id`	`String`	Version 4 UUID	The datacomponent ID
`created`	`String`	ISO 8601 timestamp	When the datacomponent was created
`lastModified`	`String`	ISO 8601 timestamp	When the datacomponent was last modified
`name`	`String`		The datacomponent name
`dataPlugin`	`String`		Create / update with dataplugin `fullyQualifiedName`
`properties`	`Properties`		The datacomponent properties, defined by the dataplugin `settings`

{
  "id" : "d13cf8c2-6f6c-4177-83c7-334733af3db7",
  "created" : "2023-03-23T21:48:50.775114",
  "lastModified" : "2023-03-23T21:48:51.014679",
  "name" : "extractors/tap-matatika-sit",
  "properties" : { },
  "commands" : { },
  "dataPlugin" : "extractors/tap-matatika-sit--matatika",
  "_embedded" : {
    "dataplugin" : {
      "id" : "8cf9c2f0-ec5f-4a82-aace-56018284fdbc",
      "pluginType" : "EXTRACTOR",
      "name" : "tap-matatika-sit",
      "namespace" : "tap_matatika_sit",
      "variant" : "matatika",
      "label" : "Matatika SIT",
      "description" : "Test extractor based on tap-spreadsheets-anywhere used during Matatika SIT runs",
      "logoUrl" : "/assets/images/datasource/tap-matatika-sit.svg",
      "hidden" : false,
      "docs" : "https://meltano.com/plugins/extractors/spreadsheets-anywhere.html",
      "pipUrl" : "git+https://github.com/ets/tap-spreadsheets-anywhere.git",
      "repo" : "https://github.com/ets/tap-spreadsheets-anywhere",
      "executable" : "tap-spreadsheets-anywhere",
      "capabilities" : [ "catalog", "state", "discover" ],
      "select" : [ ],
      "update" : { },
      "vars" : { },
      "settings" : [ {
        "name" : "tables",
        "aliases" : [ ],
        "value" : "[{\"path\":\"https://raw.githubusercontent.com/Matatika/matatika-examples/master/example_data\",\"name\":\"gitflixusers\",\"pattern\":\"GitFlixUsers.csv\",\"start_date\":\"2021-01-01T00:00:00Z\",\"key_properties\":[\"id\"],\"format\":\"csv\"}]",
        "kind" : "ARRAY",
        "description" : "An array holding json objects that each describe a set of targeted source files. See docs for details.",
        "protected" : false
      } ],
      "variants" : [ ],
      "commands" : { },
      "matatikaHidden" : false,
      "_links" : {
        "self" : {
          "href" : "https://catalog.matatika.com/api/dataplugins/8cf9c2f0-ec5f-4a82-aace-56018284fdbc"
        },
        "update dataplugin" : {
          "href" : "https://catalog.matatika.com/api/workspaces/1b07e5cc-49f2-4d12-8709-14f201f9a020/dataplugins/8cf9c2f0-ec5f-4a82-aace-56018284fdbc",
          "type" : "PUT"
        }
      }
    }
  },
  "managed" : false,
  "draft" : false,
  "_links" : {
    "self" : {
      "href" : "https://catalog.matatika.com/api/datacomponents/d13cf8c2-6f6c-4177-83c7-334733af3db7"
    },
    "update datacomponent" : {
      "href" : "https://catalog.matatika.com/api/datacomponents/d13cf8c2-6f6c-4177-83c7-334733af3db7"
    },
    "delete datacomponent" : {
      "href" : "https://catalog.matatika.com/api/datacomponents/d13cf8c2-6f6c-4177-83c7-334733af3db7"
    }
  }
}

Properties

For each setting s in the dataplugin settings:

Path	Type	Description
`s.name`	`s.kind`	Refer to `s.description`

Requests

View all datacomponents in a workspace
View a datacomponent
Initialise a new datacomponent in a workspace
Create or update a datacomponent in a workspace
Update a datacomponent
Delete a datacomponent

View all datacomponents in a workspace

GET

/api/workspaces/{workspace-id}/datacomponents

Returns all datacomponents in the workspace {workspace-id}.

Prerequisites

Workspace {workspace-id} must exist

Request

Example Snippets

cURL

curl -H "Authorization: Bearer $ACCESS_TOKEN" 'https://catalog.matatika.com:443/api/workspaces/1b07e5cc-49f2-4d12-8709-14f201f9a020/datacomponents' -i -X GET \
    -H 'Accept: application/json, application/javascript, text/javascript, text/json' \
    -H 'Content-Type: application/json'

Python (requests)

import requests

url = "https://catalog.matatika.com:443/api/workspaces/1b07e5cc-49f2-4d12-8709-14f201f9a020/datacomponents"

headers = {
  'Authorization': ACCESS_TOKEN
}

response = requests.request("GET", url, headers=headers)

print(response.text.encode('utf8'))

Response

200 OK

Datacomponent collection with HAL links.

{
  "_embedded" : {
    "datacomponents" : [ {
      "id" : "6afc5814-bed8-4239-b4f5-5cb32311fdb2",
      "created" : "2023-03-23T21:48:49.6681",
      "lastModified" : "2023-03-23T21:48:49.668101",
      "name" : "Warehouse",
      "properties" : {
        "password" : "915xHSOe_8Oda6FEm_9Nol2SPn",
        "dbname" : "afnngxm",
        "default_target_schema" : "analytics",
        "port" : "5432",
        "host" : "nutty-ugli.db.elephantsql.com",
        "user" : "afnngxm"
      },
      "commands" : { },
      "dataPlugin" : "loaders/target-postgres--transferwise",
      "_embedded" : {
        "dataplugin" : {
          "id" : "4086bc92-9917-48d5-b727-247a98e16ee4",
          "pluginType" : "LOADER",
          "name" : "target-postgres",
          "namespace" : "postgres_transferwise",
          "variant" : "transferwise",
          "label" : "Postgres Warehouse",
          "description" : "Loader for Postgres Data Warehouse\n\nThe World's Most Advanced Open Source Relational Database\n\n## Settings\n\n### User\nPostgres user\n\n### Password\nPostgres password\n\n### Host\nPostgres host\nExample: `test-host.db.elephantsql.com`\n\n### Port\nPostgres port\nExample: `5432`\n\n### Database Name\nPostgres database name\n\n### Default Target Schema\nName of the schema where tables will be created (no database prefix)",
          "logoUrl" : "/assets/logos/loaders/postgres.png",
          "hidden" : false,
          "pipUrl" : "git+https://github.com/Matatika/[email protected]",
          "repo" : "git+https://github.com/Matatika/[email protected]",
          "capabilities" : [ ],
          "select" : [ ],
          "update" : { },
          "vars" : { },
          "settings" : [ {
            "name" : "user",
            "aliases" : [ "username" ],
            "label" : "User",
            "kind" : "STRING",
            "description" : "PostgreSQL user",
            "required" : "true",
            "protected" : false
          }, {
            "name" : "password",
            "aliases" : [ ],
            "label" : "Password",
            "kind" : "PASSWORD",
            "description" : "PostgreSQL password",
            "required" : "true",
            "protected" : false
          }, {
            "name" : "host",
            "aliases" : [ "address" ],
            "label" : "Host",
            "kind" : "STRING",
            "description" : "PostgreSQL host",
            "required" : "true",
            "protected" : false
          }, {
            "name" : "port",
            "aliases" : [ ],
            "label" : "Port",
            "value" : "5432",
            "kind" : "INTEGER",
            "description" : "PostgreSQL port",
            "required" : "true",
            "protected" : false
          }, {
            "name" : "dbname",
            "aliases" : [ "database" ],
            "label" : "Database Name",
            "kind" : "STRING",
            "description" : "PostgreSQL database name",
            "required" : "true",
            "protected" : false
          }, {
            "name" : "default_target_schema",
            "aliases" : [ ],
            "label" : "Default Target Schema",
            "value" : "analytics",
            "kind" : "STRING",
            "description" : "Name of the schema where the tables will be created.",
            "required" : "true",
            "protected" : false
          }, {
            "name" : "ssl",
            "aliases" : [ ],
            "label" : "SSL",
            "value" : "false",
            "kind" : "HIDDEN",
            "protected" : false,
            "value_post_processor" : "STRINGIFY"
          }, {
            "name" : "batch_size_rows",
            "aliases" : [ ],
            "label" : "Batch Size Rows",
            "value" : "100000",
            "kind" : "INTEGER",
            "description" : "Maximum number of rows in each batch. At the end of each batch, the rows in the batch are loaded into Postgres.",
            "protected" : false
          }, {
            "name" : "underscore_camel_case_fields",
            "aliases" : [ ],
            "label" : "Underscore Camel Case Fields",
            "value" : "true",
            "kind" : "HIDDEN",
            "description" : "Enable underscoring camel case field names, make the resulting postgres column names the same as from the Meltano variant of target-postgres.",
            "protected" : false
          }, {
            "name" : "flush_all_streams",
            "aliases" : [ ],
            "label" : "Flush All Streams",
            "value" : "false",
            "kind" : "HIDDEN",
            "description" : "Flush and load every stream into Postgres when one batch is full. Warning: This may trigger the COPY command to use files with low number of records.",
            "protected" : false
          }, {
            "name" : "parallelism",
            "aliases" : [ ],
            "label" : "Parallelism",
            "value" : "0",
            "kind" : "HIDDEN",
            "description" : "The number of threads used to flush tables. 0 will create a thread for each stream, up to parallelism_max. -1 will create a thread for each CPU core. Any other positive number will create that number of threads, up to parallelism_max.",
            "protected" : false
          }, {
            "name" : "parallelism_max",
            "aliases" : [ ],
            "label" : "Max Parallelism",
            "value" : "16",
            "kind" : "HIDDEN",
            "description" : "Max number of parallel threads to use when flushing tables.",
            "protected" : false
          }, {
            "name" : "default_target_schema_select_permission",
            "aliases" : [ ],
            "label" : "Default Target Schema Select Permission",
            "kind" : "HIDDEN",
            "description" : "Grant USAGE privilege on newly created schemas and grant SELECT privilege on newly created tables to a specific role or a list of roles. If `schema_mapping` is not defined then every stream sent by the tap is granted accordingly.",
            "protected" : false
          }, {
            "name" : "schema_mapping",
            "aliases" : [ ],
            "label" : "Schema Mapping",
            "kind" : "HIDDEN",
            "description" : "Useful if you want to load multiple streams from one tap to multiple Postgres schemas.\n\nIf the tap sends the `stream_id` in `<schema_name>-<table_name>` format then this option overwrites the `default_target_schema` value.\nNote, that using `schema_mapping` you can overwrite the `default_target_schema_select_permission` value to grant SELECT permissions to different groups per schemas or optionally you can create indices automatically for the replicated tables.\n",
            "protected" : false
          }, {
            "name" : "add_metadata_columns",
            "aliases" : [ ],
            "label" : "Add Metadata Columns",
            "value" : "true",
            "kind" : "HIDDEN",
            "description" : "Metadata columns add extra row level information about data ingestions, (i.e. when was the row read in source, when was inserted or deleted in postgres etc.) Metadata columns are creating automatically by adding extra columns to the tables with a column prefix `_SDC_`. The column names are following the stitch naming conventions documented at https://www.stitchdata.com/docs/data-structure/integration-schemas#sdc-columns. Enabling metadata columns will flag the deleted rows by setting the `_SDC_DELETED_AT` metadata column. Without the `add_metadata_columns` option the deleted rows from singer taps will not be recongisable in Postgres.",
            "protected" : false
          }, {
            "name" : "hard_delete",
            "aliases" : [ ],
            "label" : "Hard Delete",
            "value" : "false",
            "kind" : "HIDDEN",
            "description" : "When `hard_delete` option is true then DELETE SQL commands will be performed in Postgres to delete rows in tables. It's achieved by continuously checking the `_SDC_DELETED_AT` metadata column sent by the singer tap. Due to deleting rows requires metadata columns, `hard_delete` option automatically enables the `add_metadata_columns` option as well.",
            "protected" : false
          }, {
            "name" : "data_flattening_max_level",
            "aliases" : [ ],
            "label" : "Data Flattening Max Level",
            "value" : "10",
            "kind" : "HIDDEN",
            "description" : "Object type RECORD items from taps can be transformed to flattened columns by creating columns automatically. When value is 0 (default) then flattening functionality is turned off.",
            "protected" : false
          }, {
            "name" : "primary_key_required",
            "aliases" : [ ],
            "label" : "Primary Key Required",
            "value" : "false",
            "kind" : "BOOLEAN",
            "description" : "Log based and Incremental replications on tables with no Primary Key cause duplicates when merging UPDATE events. When set to true, stop loading data if no Primary Key is defined.",
            "protected" : false
          }, {
            "name" : "validate_records",
            "aliases" : [ ],
            "label" : "Validate Records",
            "value" : "false",
            "kind" : "BOOLEAN",
            "description" : "Validate every single record message to the corresponding JSON schema. This option is disabled by default and invalid RECORD messages will fail only at load time by Postgres. Enabling this option will detect invalid records earlier but could cause performance degradation.",
            "protected" : false
          }, {
            "name" : "temp_dir",
            "aliases" : [ ],
            "label" : "Temporary Directory",
            "kind" : "HIDDEN",
            "description" : "(Default: platform-dependent) Directory of temporary CSV files with RECORD messages.",
            "protected" : false
          } ],
          "variants" : [ ],
          "commands" : { },
          "matatikaHidden" : false,
          "_links" : {
            "self" : {
              "href" : "https://catalog.matatika.com/api/dataplugins/4086bc92-9917-48d5-b727-247a98e16ee4"
            },
            "update dataplugin" : {
              "href" : "https://catalog.matatika.com/api/workspaces/1b07e5cc-49f2-4d12-8709-14f201f9a020/dataplugins/4086bc92-9917-48d5-b727-247a98e16ee4",
              "type" : "PUT"
            }
          }
        }
      },
      "managed" : true,
      "draft" : false,
      "_links" : {
        "self" : {
          "href" : "https://catalog.matatika.com/api/datacomponents/6afc5814-bed8-4239-b4f5-5cb32311fdb2"
        },
        "update datacomponent" : {
          "href" : "https://catalog.matatika.com/api/datacomponents/6afc5814-bed8-4239-b4f5-5cb32311fdb2"
        },
        "delete datacomponent" : {
          "href" : "https://catalog.matatika.com/api/datacomponents/6afc5814-bed8-4239-b4f5-5cb32311fdb2"
        }
      }
    }, {
      "id" : "73d2438e-a8ce-4ad0-bd76-3d7dd3f412db",
      "created" : "2023-03-23T21:48:49.778314",
      "lastModified" : "2023-03-23T21:48:49.778315",
      "name" : "dbt",
      "properties" : { },
      "commands" : {
        "seed" : {
          "args" : "seed",
          "description" : "Load data from csv files into your data warehouse."
        },
        "compile" : {
          "args" : "compile",
          "description" : "Generates executable SQL from source model, test, and analysis files. Compiled SQL files are written to the target/ directory."
        },
        "test" : {
          "args" : "test",
          "description" : "Runs tests on data in deployed models."
        },
        "run" : {
          "args" : "run",
          "description" : "Compile SQL and execute against the current target database."
        },
        "deps" : {
          "args" : "deps",
          "description" : "Pull the most recent version of the dependencies listed in packages.yml"
        },
        "clean" : {
          "args" : "clean",
          "description" : "Delete all folders in the clean-targets list (usually the dbt_modules and target directories.)"
        },
        "snapshot" : {
          "args" : "snapshot",
          "description" : "Execute snapshots defined in your project."
        }
      },
      "dataPlugin" : "transformers/dbt--dbt-labs",
      "_embedded" : {
        "dataplugin" : {
          "id" : "3a946b9d-08ea-4ef2-92b5-65da77ae8759",
          "pluginType" : "TRANSFORMER",
          "name" : "dbt",
          "namespace" : "dbt",
          "variant" : "dbt-labs",
          "label" : "dbt",
          "logoUrl" : "/assets/images/transformer/dbt.png",
          "hidden" : false,
          "docs" : "https://meltano.com/docs/transforms.html",
          "pipUrl" : "dbt-core~=1.3.0 dbt-postgres~=1.3.0 dbt-snowflake~=1.3.0\n",
          "repo" : "https://github.com/dbt-labs/dbt-core",
          "capabilities" : [ ],
          "select" : [ ],
          "update" : { },
          "vars" : { },
          "settings" : [ {
            "name" : "project_dir",
            "aliases" : [ ],
            "value" : "$MELTANO_PROJECT_ROOT/transform",
            "kind" : "STRING",
            "protected" : false
          }, {
            "name" : "profiles_dir",
            "aliases" : [ ],
            "value" : "$MELTANO_PROJECT_ROOT/transform/profile",
            "kind" : "STRING",
            "env" : "DBT_PROFILES_DIR",
            "protected" : false
          }, {
            "name" : "target",
            "aliases" : [ ],
            "value" : "$MELTANO_LOAD__DIALECT",
            "kind" : "STRING",
            "protected" : false
          }, {
            "name" : "source_schema",
            "aliases" : [ ],
            "value" : "$MELTANO_LOAD__TARGET_SCHEMA",
            "kind" : "STRING",
            "protected" : false
          }, {
            "name" : "target_schema",
            "aliases" : [ ],
            "value" : "analytics",
            "kind" : "STRING",
            "protected" : false
          }, {
            "name" : "models",
            "aliases" : [ ],
            "value" : "$MELTANO_TRANSFORM__PACKAGE_NAME $MELTANO_EXTRACTOR_NAMESPACE my_meltano_project",
            "kind" : "STRING",
            "protected" : false
          } ],
          "variants" : [ ],
          "commands" : {
            "seed" : {
              "args" : "seed",
              "description" : "Load data from csv files into your data warehouse."
            },
            "compile" : {
              "args" : "compile",
              "description" : "Generates executable SQL from source model, test, and analysis files. Compiled SQL files are written to the target/ directory."
            },
            "test" : {
              "args" : "test",
              "description" : "Runs tests on data in deployed models."
            },
            "run" : {
              "args" : "run",
              "description" : "Compile SQL and execute against the current target database."
            },
            "deps" : {
              "args" : "deps",
              "description" : "Pull the most recent version of the dependencies listed in packages.yml"
            },
            "clean" : {
              "args" : "clean",
              "description" : "Delete all folders in the clean-targets list (usually the dbt_modules and target directories.)"
            },
            "snapshot" : {
              "args" : "snapshot",
              "description" : "Execute snapshots defined in your project."
            }
          },
          "matatikaHidden" : false,
          "_links" : {
            "self" : {
              "href" : "https://catalog.matatika.com/api/dataplugins/3a946b9d-08ea-4ef2-92b5-65da77ae8759"
            },
            "update dataplugin" : {
              "href" : "https://catalog.matatika.com/api/workspaces/1b07e5cc-49f2-4d12-8709-14f201f9a020/dataplugins/3a946b9d-08ea-4ef2-92b5-65da77ae8759",
              "type" : "PUT"
            }
          }
        }
      },
      "managed" : true,
      "draft" : false,
      "_links" : {
        "self" : {
          "href" : "https://catalog.matatika.com/api/datacomponents/73d2438e-a8ce-4ad0-bd76-3d7dd3f412db"
        },
        "update datacomponent" : {
          "href" : "https://catalog.matatika.com/api/datacomponents/73d2438e-a8ce-4ad0-bd76-3d7dd3f412db"
        },
        "delete datacomponent" : {
          "href" : "https://catalog.matatika.com/api/datacomponents/73d2438e-a8ce-4ad0-bd76-3d7dd3f412db"
        }
      }
    }, {
      "id" : "d13cf8c2-6f6c-4177-83c7-334733af3db7",
      "created" : "2023-03-23T21:48:50.775114",
      "lastModified" : "2023-03-23T21:48:51.014679",
      "name" : "extractors/tap-matatika-sit",
      "properties" : { },
      "commands" : { },
      "dataPlugin" : "extractors/tap-matatika-sit--matatika",
      "_embedded" : {
        "dataplugin" : {
          "id" : "8cf9c2f0-ec5f-4a82-aace-56018284fdbc",
          "pluginType" : "EXTRACTOR",
          "name" : "tap-matatika-sit",
          "namespace" : "tap_matatika_sit",
          "variant" : "matatika",
          "label" : "Matatika SIT",
          "description" : "Test extractor based on tap-spreadsheets-anywhere used during Matatika SIT runs",
          "logoUrl" : "/assets/images/datasource/tap-matatika-sit.svg",
          "hidden" : false,
          "docs" : "https://meltano.com/plugins/extractors/spreadsheets-anywhere.html",
          "pipUrl" : "git+https://github.com/ets/tap-spreadsheets-anywhere.git",
          "repo" : "https://github.com/ets/tap-spreadsheets-anywhere",
          "executable" : "tap-spreadsheets-anywhere",
          "capabilities" : [ "catalog", "state", "discover" ],
          "select" : [ ],
          "update" : { },
          "vars" : { },
          "settings" : [ {
            "name" : "tables",
            "aliases" : [ ],
            "value" : "[{\"path\":\"https://raw.githubusercontent.com/Matatika/matatika-examples/master/example_data\",\"name\":\"gitflixusers\",\"pattern\":\"GitFlixUsers.csv\",\"start_date\":\"2021-01-01T00:00:00Z\",\"key_properties\":[\"id\"],\"format\":\"csv\"}]",
            "kind" : "ARRAY",
            "description" : "An array holding json objects that each describe a set of targeted source files. See docs for details.",
            "protected" : false
          } ],
          "variants" : [ ],
          "commands" : { },
          "matatikaHidden" : false,
          "_links" : {
            "self" : {
              "href" : "https://catalog.matatika.com/api/dataplugins/8cf9c2f0-ec5f-4a82-aace-56018284fdbc"
            },
            "update dataplugin" : {
              "href" : "https://catalog.matatika.com/api/workspaces/1b07e5cc-49f2-4d12-8709-14f201f9a020/dataplugins/8cf9c2f0-ec5f-4a82-aace-56018284fdbc",
              "type" : "PUT"
            }
          }
        }
      },
      "managed" : false,
      "draft" : false,
      "_links" : {
        "self" : {
          "href" : "https://catalog.matatika.com/api/datacomponents/d13cf8c2-6f6c-4177-83c7-334733af3db7"
        },
        "update datacomponent" : {
          "href" : "https://catalog.matatika.com/api/datacomponents/d13cf8c2-6f6c-4177-83c7-334733af3db7"
        },
        "delete datacomponent" : {
          "href" : "https://catalog.matatika.com/api/datacomponents/d13cf8c2-6f6c-4177-83c7-334733af3db7"
        }
      }
    } ]
  },
  "_links" : {
    "self" : {
      "href" : "https://catalog.matatika.com/api/workspaces/1b07e5cc-49f2-4d12-8709-14f201f9a020/datacomponents?page=0&size=20"
    }
  },
  "page" : {
    "size" : 20,
    "totalElements" : 3,
    "totalPages" : 1,
    "number" : 0
  }
}

View a datacomponent

GET

/api/datacomponents/{datacomponent-id}

Returns the datacomponent {datacomponent-id}.

Prerequisites

Datacomponent {datacomponent-id} must exist

Request

Example Snippets

cURL

curl -H "Authorization: Bearer $ACCESS_TOKEN" 'https://catalog.matatika.com:443/api/datacomponents/d13cf8c2-6f6c-4177-83c7-334733af3db7' -i -X GET \
    -H 'Accept: application/json, application/javascript, text/javascript, text/json' \
    -H 'Content-Type: application/json'

Python (requests)

import requests

url = "https://catalog.matatika.com:443/api/datacomponents/d13cf8c2-6f6c-4177-83c7-334733af3db7"

headers = {
  'Authorization': ACCESS_TOKEN
}

response = requests.request("GET", url, headers=headers)

print(response.text.encode('utf8'))

Response

200 OK

Datacomponent with HAL links.

{
  "id" : "d13cf8c2-6f6c-4177-83c7-334733af3db7",
  "created" : "2023-03-23T21:48:50.775114",
  "lastModified" : "2023-03-23T21:48:51.014679",
  "name" : "extractors/tap-matatika-sit",
  "properties" : { },
  "commands" : { },
  "dataPlugin" : "extractors/tap-matatika-sit--matatika",
  "_embedded" : {
    "dataplugin" : {
      "id" : "8cf9c2f0-ec5f-4a82-aace-56018284fdbc",
      "pluginType" : "EXTRACTOR",
      "name" : "tap-matatika-sit",
      "namespace" : "tap_matatika_sit",
      "variant" : "matatika",
      "label" : "Matatika SIT",
      "description" : "Test extractor based on tap-spreadsheets-anywhere used during Matatika SIT runs",
      "logoUrl" : "/assets/images/datasource/tap-matatika-sit.svg",
      "hidden" : false,
      "docs" : "https://meltano.com/plugins/extractors/spreadsheets-anywhere.html",
      "pipUrl" : "git+https://github.com/ets/tap-spreadsheets-anywhere.git",
      "repo" : "https://github.com/ets/tap-spreadsheets-anywhere",
      "executable" : "tap-spreadsheets-anywhere",
      "capabilities" : [ "catalog", "state", "discover" ],
      "select" : [ ],
      "update" : { },
      "vars" : { },
      "settings" : [ {
        "name" : "tables",
        "aliases" : [ ],
        "value" : "[{\"path\":\"https://raw.githubusercontent.com/Matatika/matatika-examples/master/example_data\",\"name\":\"gitflixusers\",\"pattern\":\"GitFlixUsers.csv\",\"start_date\":\"2021-01-01T00:00:00Z\",\"key_properties\":[\"id\"],\"format\":\"csv\"}]",
        "kind" : "ARRAY",
        "description" : "An array holding json objects that each describe a set of targeted source files. See docs for details.",
        "protected" : false
      } ],
      "variants" : [ ],
      "commands" : { },
      "matatikaHidden" : false,
      "_links" : {
        "self" : {
          "href" : "https://catalog.matatika.com/api/dataplugins/8cf9c2f0-ec5f-4a82-aace-56018284fdbc"
        },
        "update dataplugin" : {
          "href" : "https://catalog.matatika.com/api/workspaces/1b07e5cc-49f2-4d12-8709-14f201f9a020/dataplugins/8cf9c2f0-ec5f-4a82-aace-56018284fdbc",
          "type" : "PUT"
        }
      }
    }
  },
  "managed" : false,
  "draft" : false,
  "_links" : {
    "self" : {
      "href" : "https://catalog.matatika.com/api/datacomponents/d13cf8c2-6f6c-4177-83c7-334733af3db7"
    },
    "update datacomponent" : {
      "href" : "https://catalog.matatika.com/api/datacomponents/d13cf8c2-6f6c-4177-83c7-334733af3db7"
    },
    "delete datacomponent" : {
      "href" : "https://catalog.matatika.com/api/datacomponents/d13cf8c2-6f6c-4177-83c7-334733af3db7"
    }
  }
}

Initialise a new datacomponent in a workspace

POST

/api/workspaces/{workspace-id}/datacomponents

Initialises a new datacomponent in the workspace {workspace-id}.

Request

Example Snippets

cURL

curl -H "Authorization: Bearer $ACCESS_TOKEN" 'https://catalog.matatika.com:443/api/workspaces/1b07e5cc-49f2-4d12-8709-14f201f9a020/datacomponents' -i -X POST \
    -H 'Accept: application/json, application/javascript, text/javascript, text/json' \
    -H 'Content-Type: application/json'

Python (requests)

import requests

url = "https://catalog.matatika.com:443/api/workspaces/1b07e5cc-49f2-4d12-8709-14f201f9a020/datacomponents"

headers = {
  'Authorization': ACCESS_TOKEN
}

response = requests.request("POST", url, headers=headers)

print(response.text.encode('utf8'))

Response

200 OK

datacomponent with HAL links.

{
  "id" : "d13cf8c2-6f6c-4177-83c7-334733af3db7",
  "created" : "2023-03-23T21:48:50.386396995",
  "lastModified" : "2023-03-23T21:48:50.386397495",
  "properties" : { },
  "commands" : { },
  "managed" : false,
  "draft" : false,
  "_links" : {
    "self" : {
      "href" : "https://catalog.matatika.com/api/datacomponents/d13cf8c2-6f6c-4177-83c7-334733af3db7"
    },
    "create datacomponent" : {
      "href" : "https://catalog.matatika.com/api/workspaces/1b07e5cc-49f2-4d12-8709-14f201f9a020/datacomponents/d13cf8c2-6f6c-4177-83c7-334733af3db7"
    }
  }
}

Create or update a datacomponent in a workspace

PUT

/api/workspaces/{workspace-id}/datacomponents/{datacomponent-id}

Creates or updates the datacomponent {datacomponent-id} in the workspace {workspace-id}.

Request

Body

Datacomponent resource.

{
  "name" : "extractors/tap-matatika-sit",
  "dataPlugin" : "extractors/tap-matatika-sit"
}

Example Snippets

cURL

curl -H "Authorization: Bearer $ACCESS_TOKEN" 'https://catalog.matatika.com:443/api/workspaces/1b07e5cc-49f2-4d12-8709-14f201f9a020/datacomponents/d13cf8c2-6f6c-4177-83c7-334733af3db7' -i -X PUT \
    -H 'Accept: application/json, application/javascript, text/javascript, text/json' \
    -H 'Content-Type: application/hal+json; charset=ISO-8859-1' \
    -d '{
  "name" : "extractors/tap-matatika-sit",
  "dataPlugin" : "extractors/tap-matatika-sit"
}'

Python (requests)

import requests

url = "https://catalog.matatika.com:443/api/workspaces/1b07e5cc-49f2-4d12-8709-14f201f9a020/datacomponents/d13cf8c2-6f6c-4177-83c7-334733af3db7"

data = {
  "name" : "extractors/tap-matatika-sit",
  "dataPlugin" : "extractors/tap-matatika-sit"
}
headers = {
  'Authorization': ACCESS_TOKEN
}

response = requests.request("PUT", url, headers=headers, data=data)

print(response.text.encode('utf8'))

Response

200 OK / 201 Created

Datacomponent with HAL links.

{
  "id" : "d13cf8c2-6f6c-4177-83c7-334733af3db7",
  "created" : "2023-03-23T21:48:50.775114",
  "lastModified" : "2023-03-23T21:48:50.775115",
  "name" : "extractors/tap-matatika-sit",
  "properties" : { },
  "commands" : { },
  "dataPlugin" : "extractors/tap-matatika-sit--matatika",
  "_embedded" : {
    "dataplugin" : {
      "id" : "8cf9c2f0-ec5f-4a82-aace-56018284fdbc",
      "pluginType" : "EXTRACTOR",
      "name" : "tap-matatika-sit",
      "namespace" : "tap_matatika_sit",
      "variant" : "matatika",
      "label" : "Matatika SIT",
      "description" : "Test extractor based on tap-spreadsheets-anywhere used during Matatika SIT runs",
      "logoUrl" : "/assets/images/datasource/tap-matatika-sit.svg",
      "hidden" : false,
      "docs" : "https://meltano.com/plugins/extractors/spreadsheets-anywhere.html",
      "pipUrl" : "git+https://github.com/ets/tap-spreadsheets-anywhere.git",
      "repo" : "https://github.com/ets/tap-spreadsheets-anywhere",
      "executable" : "tap-spreadsheets-anywhere",
      "capabilities" : [ "catalog", "state", "discover" ],
      "select" : [ ],
      "update" : { },
      "vars" : { },
      "settings" : [ {
        "name" : "tables",
        "aliases" : [ ],
        "value" : "[{\"path\":\"https://raw.githubusercontent.com/Matatika/matatika-examples/master/example_data\",\"name\":\"gitflixusers\",\"pattern\":\"GitFlixUsers.csv\",\"start_date\":\"2021-01-01T00:00:00Z\",\"key_properties\":[\"id\"],\"format\":\"csv\"}]",
        "kind" : "ARRAY",
        "description" : "An array holding json objects that each describe a set of targeted source files. See docs for details.",
        "protected" : false
      } ],
      "variants" : [ ],
      "commands" : { },
      "matatikaHidden" : false,
      "_links" : {
        "self" : {
          "href" : "https://catalog.matatika.com/api/dataplugins/8cf9c2f0-ec5f-4a82-aace-56018284fdbc"
        },
        "update dataplugin" : {
          "href" : "https://catalog.matatika.com/api/workspaces/1b07e5cc-49f2-4d12-8709-14f201f9a020/dataplugins/8cf9c2f0-ec5f-4a82-aace-56018284fdbc",
          "type" : "PUT"
        }
      }
    }
  },
  "managed" : false,
  "draft" : false,
  "_links" : {
    "self" : {
      "href" : "https://catalog.matatika.com/api/datacomponents/d13cf8c2-6f6c-4177-83c7-334733af3db7"
    },
    "update datacomponent" : {
      "href" : "https://catalog.matatika.com/api/datacomponents/d13cf8c2-6f6c-4177-83c7-334733af3db7"
    },
    "delete datacomponent" : {
      "href" : "https://catalog.matatika.com/api/datacomponents/d13cf8c2-6f6c-4177-83c7-334733af3db7"
    }
  }
}

Update a datacomponent

PUT

/api/datacomponents/{datacomponent-id}

Updates the datacomponent {datacomponent-id}.

Prerequisites

Datacomponent {datacomponent-id} must exist

Request

Body

Datacomponent resource.

{
  "name" : "extractors/tap-matatika-sit",
  "dataPlugin" : "extractors/tap-matatika-sit"
}

Example Snippets

cURL

curl -H "Authorization: Bearer $ACCESS_TOKEN" 'https://catalog.matatika.com:443/api/datacomponents/d13cf8c2-6f6c-4177-83c7-334733af3db7' -i -X PUT \
    -H 'Accept: application/json, application/javascript, text/javascript, text/json' \
    -H 'Content-Type: application/hal+json; charset=ISO-8859-1' \
    -d '{
  "name" : "extractors/tap-matatika-sit",
  "dataPlugin" : "extractors/tap-matatika-sit"
}'

Python (requests)

import requests

url = "https://catalog.matatika.com:443/api/datacomponents/d13cf8c2-6f6c-4177-83c7-334733af3db7"

data = {
  "name" : "extractors/tap-matatika-sit",
  "dataPlugin" : "extractors/tap-matatika-sit"
}
headers = {
  'Authorization': ACCESS_TOKEN
}

response = requests.request("PUT", url, headers=headers, data=data)

print(response.text.encode('utf8'))

Response

200 OK

Datacomponent with HAL links.

{
  "id" : "d13cf8c2-6f6c-4177-83c7-334733af3db7",
  "created" : "2023-03-23T21:48:50.775114",
  "lastModified" : "2023-03-23T21:48:51.014679",
  "name" : "extractors/tap-matatika-sit",
  "properties" : { },
  "commands" : { },
  "dataPlugin" : "extractors/tap-matatika-sit--matatika",
  "_embedded" : {
    "dataplugin" : {
      "id" : "8cf9c2f0-ec5f-4a82-aace-56018284fdbc",
      "pluginType" : "EXTRACTOR",
      "name" : "tap-matatika-sit",
      "namespace" : "tap_matatika_sit",
      "variant" : "matatika",
      "label" : "Matatika SIT",
      "description" : "Test extractor based on tap-spreadsheets-anywhere used during Matatika SIT runs",
      "logoUrl" : "/assets/images/datasource/tap-matatika-sit.svg",
      "hidden" : false,
      "docs" : "https://meltano.com/plugins/extractors/spreadsheets-anywhere.html",
      "pipUrl" : "git+https://github.com/ets/tap-spreadsheets-anywhere.git",
      "repo" : "https://github.com/ets/tap-spreadsheets-anywhere",
      "executable" : "tap-spreadsheets-anywhere",
      "capabilities" : [ "catalog", "state", "discover" ],
      "select" : [ ],
      "update" : { },
      "vars" : { },
      "settings" : [ {
        "name" : "tables",
        "aliases" : [ ],
        "value" : "[{\"path\":\"https://raw.githubusercontent.com/Matatika/matatika-examples/master/example_data\",\"name\":\"gitflixusers\",\"pattern\":\"GitFlixUsers.csv\",\"start_date\":\"2021-01-01T00:00:00Z\",\"key_properties\":[\"id\"],\"format\":\"csv\"}]",
        "kind" : "ARRAY",
        "description" : "An array holding json objects that each describe a set of targeted source files. See docs for details.",
        "protected" : false
      } ],
      "variants" : [ ],
      "commands" : { },
      "matatikaHidden" : false,
      "_links" : {
        "self" : {
          "href" : "https://catalog.matatika.com/api/dataplugins/8cf9c2f0-ec5f-4a82-aace-56018284fdbc"
        },
        "update dataplugin" : {
          "href" : "https://catalog.matatika.com/api/workspaces/1b07e5cc-49f2-4d12-8709-14f201f9a020/dataplugins/8cf9c2f0-ec5f-4a82-aace-56018284fdbc",
          "type" : "PUT"
        }
      }
    }
  },
  "managed" : false,
  "draft" : false,
  "_links" : {
    "self" : {
      "href" : "https://catalog.matatika.com/api/datacomponents/d13cf8c2-6f6c-4177-83c7-334733af3db7"
    },
    "update datacomponent" : {
      "href" : "https://catalog.matatika.com/api/datacomponents/d13cf8c2-6f6c-4177-83c7-334733af3db7"
    },
    "delete datacomponent" : {
      "href" : "https://catalog.matatika.com/api/datacomponents/d13cf8c2-6f6c-4177-83c7-334733af3db7"
    }
  }
}

Delete a datacomponent

DELETE

/api/datacomponents/{datacomponent-id}

Deletes the datacomponent {datacomponent-id}.

Request

Example Snippets

cURL

curl -H "Authorization: Bearer $ACCESS_TOKEN" 'https://catalog.matatika.com:443/api/datacomponents/d13cf8c2-6f6c-4177-83c7-334733af3db7' -i -X DELETE \
    -H 'Accept: application/json, application/javascript, text/javascript, text/json' \
    -H 'Content-Type: application/json'

Python (requests)

import requests

url = "https://catalog.matatika.com:443/api/datacomponents/d13cf8c2-6f6c-4177-83c7-334733af3db7"

headers = {
  'Authorization': ACCESS_TOKEN
}

response = requests.request("DELETE", url, headers=headers)

print(response.text.encode('utf8'))

Response

204 No Content

No response body provided.