Link Search Menu Expand Document

Datacomponents

Datacomponents hold configuration for dataplugins, and are the building blocks for constructing pipelines. One dataplugin may be referenced by many datacomponents, each with a different set of properties for the dataplugin settings. One pipeline may reference multiple datacomponents.


Objects

Datacomponent

Path Type Format Description
id String Version 4 UUID The datacomponent ID
created String ISO 8601 timestamp When the datacomponent was created
lastModified String ISO 8601 timestamp When the datacomponent was last modified
name String   The datacomponent name
dataPlugin String Dataplugin name The datacomponent dataplugin name
properties Properties   The datacomponent properties, defined by the dataplugin settings
{
  "id" : "adcafd34-cc1b-4a78-abc6-81f4c7e55b6b",
  "created" : "2022-09-23T12:43:00.438046",
  "lastModified" : "2022-09-23T12:43:00.646195",
  "name" : "tap-matatika-sit",
  "properties" : { },
  "_embedded" : {
    "dataplugin" : {
      "id" : "fb2271ad-7a7c-4b8c-9dba-b3bcfada2aa2",
      "type" : "EXTRACTOR",
      "name" : "tap-matatika-sit",
      "namespace" : "tap_matatika_sit",
      "label" : "Matatika SIT",
      "description" : "Test extractor based on tap-spreadsheets-anywhere used during Matatika SIT runs",
      "logoUrl" : "/assets/images/datasource/tap-matatika-sit.svg",
      "hidden" : false,
      "documentationUrl" : "https://meltano.com/plugins/extractors/spreadsheets-anywhere.html",
      "pipUrl" : "git+https://github.com/ets/tap-spreadsheets-anywhere.git",
      "repositoryUrl" : "https://github.com/ets/tap-spreadsheets-anywhere",
      "executable" : "tap-spreadsheets-anywhere",
      "capabilities" : [ ],
      "select" : [ ],
      "update" : { },
      "vars" : { },
      "settings" : [ {
        "name" : "tables",
        "aliases" : [ ],
        "label" : "Tables",
        "value" : "[{\"path\":\"https://raw.githubusercontent.com/Matatika/matatika-examples/master/example_data\",\"name\":\"gitflixusers\",\"pattern\":\"GitFlixUsers.csv\",\"start_date\":\"2021-01-01T00:00:00Z\",\"key_properties\":[\"id\"],\"format\":\"csv\"}]",
        "kind" : "ARRAY",
        "description" : "An array holding json objects that each describe a set of targeted source files. See docs for details.",
        "protected" : false
      } ],
      "commands" : { },
      "matatikaHidden" : true,
      "_links" : {
        "self" : {
          "href" : "https://catalog.matatika.com/api/dataplugins/fb2271ad-7a7c-4b8c-9dba-b3bcfada2aa2"
        },
        "update dataplugin" : {
          "href" : "https://catalog.matatika.com/api/workspaces/2583dede-e0c3-45c0-8d5d-2e85f5de200a/dataplugins/fb2271ad-7a7c-4b8c-9dba-b3bcfada2aa2",
          "type" : "PUT"
        }
      }
    }
  },
  "managed" : false,
  "_links" : {
    "self" : {
      "href" : "https://catalog.matatika.com/api/datacomponents/adcafd34-cc1b-4a78-abc6-81f4c7e55b6b"
    },
    "update datacomponent" : {
      "href" : "https://catalog.matatika.com/api/datacomponents/adcafd34-cc1b-4a78-abc6-81f4c7e55b6b"
    },
    "delete datacomponent" : {
      "href" : "https://catalog.matatika.com/api/datacomponents/adcafd34-cc1b-4a78-abc6-81f4c7e55b6b"
    }
  }
}

Properties

For each setting s in the dataplugin settings:

Path Type Description
s.name s.kind Refer to s.description

Requests


View all datacomponents in a workspace

GET

/api/workspaces/{workspace-id}/datacomponents

Returns all datacomponents in the workspace {workspace-id}.

Prerequisites

  • Workspace {workspace-id} must exist

Request

Example Snippets

  • cURL
curl -H "Authorization: Bearer $ACCESS_TOKEN" 'https://catalog.matatika.com:443/api/workspaces/2583dede-e0c3-45c0-8d5d-2e85f5de200a/datacomponents' -i -X GET \
    -H 'Accept: application/json, application/javascript, text/javascript, text/json' \
    -H 'Content-Type: application/json'
  • Python (requests)
import requests

url = "https://catalog.matatika.com:443/api/workspaces/2583dede-e0c3-45c0-8d5d-2e85f5de200a/datacomponents"

headers = {
  'Authorization': ACCESS_TOKEN
}

response = requests.request("GET", url, headers=headers)

print(response.text.encode('utf8'))

Response

200 OK

Datacomponent collection with HAL links.

{
  "_embedded" : {
    "datacomponents" : [ {
      "id" : "9708b017-d775-4b3c-815b-9dea05cfa456",
      "created" : "2022-09-23T12:42:59.724867",
      "lastModified" : "2022-09-23T12:42:59.724868",
      "name" : "Warehouse",
      "properties" : {
        "password" : "ek6Kwl3L_I7yGBq0c_sEQp5N0s",
        "dbname" : "hmkxacy",
        "default_target_schema" : "analytics",
        "port" : "5432",
        "host" : "nutty-ugli.db.elephantsql.com",
        "user" : "hmkxacy"
      },
      "_embedded" : {
        "dataplugin" : {
          "id" : "2afd7691-f562-4ed8-8642-bf135770cc96",
          "type" : "LOADER",
          "name" : "target-postgres--transferwise",
          "namespace" : "postgres_transferwise",
          "variant" : "transferwise",
          "label" : "PostgreSQL",
          "description" : "PostgreSQL Data Warehouse\n\nThe World's Most Advanced Open Source Relational Database\n\n## Settings\n\n### User\nPostgres user\n\n### Password\nPostgres password\n\n### Host\nPostgres host\nExample: `test-host.db.elephantsql.com`\n\n### Port\nPostgres port\nExample: `5432`\n\n### Database Name\nPostgres database name\n\n### Default Target Schema\nName of the schema where tables will be created (no database prefix)",
          "logoUrl" : "/assets/images/datastore/target-postgres.png",
          "hidden" : false,
          "documentationUrl" : "https://hub.meltano.com/loaders/postgres.html",
          "pipUrl" : "git+https://github.com/Matatika/[email protected]",
          "repositoryUrl" : "https://github.com/transferwise/pipelinewise-target-postgres",
          "executable" : "target-postgres",
          "capabilities" : [ ],
          "select" : [ ],
          "update" : { },
          "vars" : { },
          "settings" : [ {
            "name" : "user",
            "aliases" : [ "username" ],
            "label" : "User",
            "kind" : "STRING",
            "description" : "PostgreSQL user",
            "protected" : false
          }, {
            "name" : "password",
            "aliases" : [ ],
            "label" : "Password",
            "kind" : "PASSWORD",
            "description" : "PostgreSQL password",
            "protected" : false
          }, {
            "name" : "host",
            "aliases" : [ "address" ],
            "label" : "Host",
            "kind" : "STRING",
            "description" : "PostgreSQL host",
            "protected" : false
          }, {
            "name" : "port",
            "aliases" : [ ],
            "label" : "Port",
            "value" : "5432",
            "kind" : "INTEGER",
            "description" : "PostgreSQL port",
            "protected" : false
          }, {
            "name" : "dbname",
            "aliases" : [ "database" ],
            "label" : "Database Name",
            "kind" : "STRING",
            "description" : "PostgreSQL database name",
            "protected" : false
          }, {
            "name" : "default_target_schema",
            "aliases" : [ ],
            "label" : "Default Target Schema",
            "value" : "analytics",
            "kind" : "STRING",
            "description" : "Name of the schema where the tables will be created.",
            "protected" : false
          }, {
            "name" : "ssl",
            "aliases" : [ ],
            "label" : "SSL",
            "value" : "false",
            "kind" : "HIDDEN",
            "valuePostProcessor" : "STRINGIFY",
            "required" : "false",
            "protected" : false
          }, {
            "name" : "batch_size_rows",
            "aliases" : [ ],
            "label" : "Batch Size Rows",
            "value" : "100000",
            "kind" : "INTEGER",
            "description" : "Maximum number of rows in each batch. At the end of each batch, the rows in the batch are loaded into Postgres.",
            "required" : "false",
            "protected" : false
          }, {
            "name" : "underscore_camel_case_fields",
            "aliases" : [ ],
            "label" : "Underscore Camel Case Fields",
            "value" : "true",
            "kind" : "HIDDEN",
            "description" : "Enable underscoring camel case field names, make the resulting postgres column names the same as from the Meltano variant of target-postgres.",
            "required" : "false",
            "protected" : false
          }, {
            "name" : "flush_all_streams",
            "aliases" : [ ],
            "label" : "Flush All Streams",
            "value" : "false",
            "kind" : "HIDDEN",
            "description" : "Flush and load every stream into Postgres when one batch is full. Warning: This may trigger the COPY command to use files with low number of records.",
            "required" : "false",
            "protected" : false
          }, {
            "name" : "parallelism",
            "aliases" : [ ],
            "label" : "Parallelism",
            "value" : "0",
            "kind" : "HIDDEN",
            "description" : "The number of threads used to flush tables. 0 will create a thread for each stream, up to parallelism_max. -1 will create a thread for each CPU core. Any other positive number will create that number of threads, up to parallelism_max.",
            "required" : "false",
            "protected" : false
          }, {
            "name" : "parallelism_max",
            "aliases" : [ ],
            "label" : "Max Parallelism",
            "value" : "16",
            "kind" : "HIDDEN",
            "description" : "Max number of parallel threads to use when flushing tables.",
            "required" : "false",
            "protected" : false
          }, {
            "name" : "default_target_schema_select_permission",
            "aliases" : [ ],
            "label" : "Default Target Schema Select Permission",
            "kind" : "HIDDEN",
            "description" : "Grant USAGE privilege on newly created schemas and grant SELECT privilege on newly created tables to a specific role or a list of roles. If `schema_mapping` is not defined then every stream sent by the tap is granted accordingly.",
            "required" : "false",
            "protected" : false
          }, {
            "name" : "schema_mapping",
            "aliases" : [ ],
            "label" : "Schema Mapping",
            "kind" : "HIDDEN",
            "description" : "Useful if you want to load multiple streams from one tap to multiple Postgres schemas.\n\nIf the tap sends the `stream_id` in `<schema_name>-<table_name>` format then this option overwrites the `default_target_schema` value.\nNote, that using `schema_mapping` you can overwrite the `default_target_schema_select_permission` value to grant SELECT permissions to different groups per schemas or optionally you can create indices automatically for the replicated tables.\n",
            "required" : "false",
            "protected" : false
          }, {
            "name" : "add_metadata_columns",
            "aliases" : [ ],
            "label" : "Add Metadata Columns",
            "value" : "true",
            "kind" : "HIDDEN",
            "description" : "Metadata columns add extra row level information about data ingestions, (i.e. when was the row read in source, when was inserted or deleted in postgres etc.) Metadata columns are creating automatically by adding extra columns to the tables with a column prefix `_SDC_`. The column names are following the stitch naming conventions documented at https://www.stitchdata.com/docs/data-structure/integration-schemas#sdc-columns. Enabling metadata columns will flag the deleted rows by setting the `_SDC_DELETED_AT` metadata column. Without the `add_metadata_columns` option the deleted rows from singer taps will not be recongisable in Postgres.",
            "required" : "false",
            "protected" : false
          }, {
            "name" : "hard_delete",
            "aliases" : [ ],
            "label" : "Hard Delete",
            "value" : "false",
            "kind" : "HIDDEN",
            "description" : "When `hard_delete` option is true then DELETE SQL commands will be performed in Postgres to delete rows in tables. It's achieved by continuously checking the `_SDC_DELETED_AT` metadata column sent by the singer tap. Due to deleting rows requires metadata columns, `hard_delete` option automatically enables the `add_metadata_columns` option as well.",
            "required" : "false",
            "protected" : false
          }, {
            "name" : "data_flattening_max_level",
            "aliases" : [ ],
            "label" : "Data Flattening Max Level",
            "value" : "10",
            "kind" : "HIDDEN",
            "description" : "Object type RECORD items from taps can be transformed to flattened columns by creating columns automatically. When value is 0 (default) then flattening functionality is turned off.",
            "required" : "false",
            "protected" : false
          }, {
            "name" : "primary_key_required",
            "aliases" : [ ],
            "label" : "Primary Key Required",
            "value" : "false",
            "kind" : "BOOLEAN",
            "description" : "Log based and Incremental replications on tables with no Primary Key cause duplicates when merging UPDATE events. When set to true, stop loading data if no Primary Key is defined.",
            "required" : "false",
            "protected" : false
          }, {
            "name" : "validate_records",
            "aliases" : [ ],
            "label" : "Validate Records",
            "value" : "false",
            "kind" : "BOOLEAN",
            "description" : "Validate every single record message to the corresponding JSON schema. This option is disabled by default and invalid RECORD messages will fail only at load time by Postgres. Enabling this option will detect invalid records earlier but could cause performance degradation.",
            "required" : "false",
            "protected" : false
          }, {
            "name" : "temp_dir",
            "aliases" : [ ],
            "label" : "Temporary Directory",
            "kind" : "HIDDEN",
            "description" : "(Default: platform-dependent) Directory of temporary CSV files with RECORD messages.",
            "required" : "false",
            "protected" : false
          } ],
          "commands" : { },
          "matatikaHidden" : false,
          "_links" : {
            "self" : {
              "href" : "https://catalog.matatika.com/api/dataplugins/2afd7691-f562-4ed8-8642-bf135770cc96"
            },
            "update dataplugin" : {
              "href" : "https://catalog.matatika.com/api/workspaces/2583dede-e0c3-45c0-8d5d-2e85f5de200a/dataplugins/2afd7691-f562-4ed8-8642-bf135770cc96",
              "type" : "PUT"
            }
          }
        }
      },
      "managed" : true,
      "_links" : {
        "self" : {
          "href" : "https://catalog.matatika.com/api/datacomponents/9708b017-d775-4b3c-815b-9dea05cfa456"
        },
        "update datacomponent" : {
          "href" : "https://catalog.matatika.com/api/datacomponents/9708b017-d775-4b3c-815b-9dea05cfa456"
        },
        "delete datacomponent" : {
          "href" : "https://catalog.matatika.com/api/datacomponents/9708b017-d775-4b3c-815b-9dea05cfa456"
        }
      }
    }, {
      "id" : "e58c7107-0f7e-421e-a20f-4f02b4abe129",
      "created" : "2022-09-23T12:42:59.753848",
      "lastModified" : "2022-09-23T12:42:59.753848",
      "name" : "dbt",
      "properties" : { },
      "_embedded" : {
        "dataplugin" : {
          "id" : "7f53c3fa-6c66-4456-b063-48d33c8681ca",
          "type" : "TRANSFORMER",
          "name" : "dbt",
          "namespace" : "dbt",
          "label" : "Dbt",
          "logoUrl" : "/assets/images/transformer/dbt.png",
          "hidden" : false,
          "documentationUrl" : "https://meltano.com/docs/transforms.html",
          "pipUrl" : "dbt-core~=1.0.0 dbt-postgres~=1.0.0 dbt-redshift~=1.0.0 dbt-snowflake~=1.0.0 dbt-bigquery~=1.0.0\n",
          "repositoryUrl" : "https://github.com/dbt-labs/dbt-core",
          "capabilities" : [ ],
          "select" : [ ],
          "update" : { },
          "vars" : { },
          "settings" : [ {
            "name" : "project_dir",
            "aliases" : [ ],
            "label" : "Project_dir",
            "value" : "$MELTANO_PROJECT_ROOT/transform",
            "kind" : "STRING",
            "protected" : false
          }, {
            "name" : "profiles_dir",
            "aliases" : [ ],
            "label" : "Profiles_dir",
            "value" : "$MELTANO_PROJECT_ROOT/transform/profile",
            "kind" : "STRING",
            "env" : "DBT_PROFILES_DIR",
            "protected" : false
          }, {
            "name" : "target",
            "aliases" : [ ],
            "label" : "Target",
            "value" : "$MELTANO_LOAD__DIALECT",
            "kind" : "STRING",
            "protected" : false
          }, {
            "name" : "source_schema",
            "aliases" : [ ],
            "label" : "Source_schema",
            "value" : "$MELTANO_LOAD__TARGET_SCHEMA",
            "kind" : "STRING",
            "protected" : false
          }, {
            "name" : "target_schema",
            "aliases" : [ ],
            "label" : "Target_schema",
            "value" : "analytics",
            "kind" : "STRING",
            "protected" : false
          }, {
            "name" : "models",
            "aliases" : [ ],
            "label" : "Models",
            "value" : "$MELTANO_TRANSFORM__PACKAGE_NAME $MELTANO_EXTRACTOR_NAMESPACE my_meltano_project",
            "kind" : "STRING",
            "protected" : false
          } ],
          "commands" : {
            "compile" : {
              "args" : "compile",
              "description" : "Generates executable SQL from source model, test, and analysis files. Compiled SQL files are written to the target/ directory."
            },
            "seed" : {
              "args" : "seed",
              "description" : "Load data from csv files into your data warehouse."
            },
            "test" : {
              "args" : "test",
              "description" : "Runs tests on data in deployed models."
            },
            "deps" : {
              "args" : "deps",
              "description" : "Pull the most recent version of the dependencies listed in packages.yml"
            },
            "run" : {
              "args" : "run",
              "description" : "Compile SQL and execute against the current target database."
            },
            "clean" : {
              "args" : "clean",
              "description" : "Delete all folders in the clean-targets list (usually the dbt_modules and target directories.)"
            },
            "snapshot" : {
              "args" : "snapshot",
              "description" : "Execute snapshots defined in your project."
            }
          },
          "matatikaHidden" : false,
          "_links" : {
            "self" : {
              "href" : "https://catalog.matatika.com/api/dataplugins/7f53c3fa-6c66-4456-b063-48d33c8681ca"
            },
            "update dataplugin" : {
              "href" : "https://catalog.matatika.com/api/workspaces/2583dede-e0c3-45c0-8d5d-2e85f5de200a/dataplugins/7f53c3fa-6c66-4456-b063-48d33c8681ca",
              "type" : "PUT"
            }
          }
        }
      },
      "managed" : true,
      "_links" : {
        "self" : {
          "href" : "https://catalog.matatika.com/api/datacomponents/e58c7107-0f7e-421e-a20f-4f02b4abe129"
        },
        "update datacomponent" : {
          "href" : "https://catalog.matatika.com/api/datacomponents/e58c7107-0f7e-421e-a20f-4f02b4abe129"
        },
        "delete datacomponent" : {
          "href" : "https://catalog.matatika.com/api/datacomponents/e58c7107-0f7e-421e-a20f-4f02b4abe129"
        }
      }
    }, {
      "id" : "adcafd34-cc1b-4a78-abc6-81f4c7e55b6b",
      "created" : "2022-09-23T12:43:00.438046",
      "lastModified" : "2022-09-23T12:43:00.646195",
      "name" : "tap-matatika-sit",
      "properties" : { },
      "_embedded" : {
        "dataplugin" : {
          "id" : "fb2271ad-7a7c-4b8c-9dba-b3bcfada2aa2",
          "type" : "EXTRACTOR",
          "name" : "tap-matatika-sit",
          "namespace" : "tap_matatika_sit",
          "label" : "Matatika SIT",
          "description" : "Test extractor based on tap-spreadsheets-anywhere used during Matatika SIT runs",
          "logoUrl" : "/assets/images/datasource/tap-matatika-sit.svg",
          "hidden" : false,
          "documentationUrl" : "https://meltano.com/plugins/extractors/spreadsheets-anywhere.html",
          "pipUrl" : "git+https://github.com/ets/tap-spreadsheets-anywhere.git",
          "repositoryUrl" : "https://github.com/ets/tap-spreadsheets-anywhere",
          "executable" : "tap-spreadsheets-anywhere",
          "capabilities" : [ ],
          "select" : [ ],
          "update" : { },
          "vars" : { },
          "settings" : [ {
            "name" : "tables",
            "aliases" : [ ],
            "label" : "Tables",
            "value" : "[{\"path\":\"https://raw.githubusercontent.com/Matatika/matatika-examples/master/example_data\",\"name\":\"gitflixusers\",\"pattern\":\"GitFlixUsers.csv\",\"start_date\":\"2021-01-01T00:00:00Z\",\"key_properties\":[\"id\"],\"format\":\"csv\"}]",
            "kind" : "ARRAY",
            "description" : "An array holding json objects that each describe a set of targeted source files. See docs for details.",
            "protected" : false
          } ],
          "commands" : { },
          "matatikaHidden" : true,
          "_links" : {
            "self" : {
              "href" : "https://catalog.matatika.com/api/dataplugins/fb2271ad-7a7c-4b8c-9dba-b3bcfada2aa2"
            },
            "update dataplugin" : {
              "href" : "https://catalog.matatika.com/api/workspaces/2583dede-e0c3-45c0-8d5d-2e85f5de200a/dataplugins/fb2271ad-7a7c-4b8c-9dba-b3bcfada2aa2",
              "type" : "PUT"
            }
          }
        }
      },
      "managed" : false,
      "_links" : {
        "self" : {
          "href" : "https://catalog.matatika.com/api/datacomponents/adcafd34-cc1b-4a78-abc6-81f4c7e55b6b"
        },
        "update datacomponent" : {
          "href" : "https://catalog.matatika.com/api/datacomponents/adcafd34-cc1b-4a78-abc6-81f4c7e55b6b"
        },
        "delete datacomponent" : {
          "href" : "https://catalog.matatika.com/api/datacomponents/adcafd34-cc1b-4a78-abc6-81f4c7e55b6b"
        }
      }
    } ]
  },
  "_links" : {
    "self" : {
      "href" : "https://catalog.matatika.com/api/workspaces/2583dede-e0c3-45c0-8d5d-2e85f5de200a/datacomponents?page=0&size=20"
    }
  },
  "page" : {
    "size" : 20,
    "totalElements" : 3,
    "totalPages" : 1,
    "number" : 0
  }
}

View a datacomponent

GET

/api/datacomponents/{datacomponent-id}

Returns the datacomponent {datacomponent-id}.

Prerequisites

  • Datacomponent {datacomponent-id} must exist

Request

Example Snippets

  • cURL
curl -H "Authorization: Bearer $ACCESS_TOKEN" 'https://catalog.matatika.com:443/api/datacomponents/adcafd34-cc1b-4a78-abc6-81f4c7e55b6b' -i -X GET \
    -H 'Accept: application/json, application/javascript, text/javascript, text/json' \
    -H 'Content-Type: application/json'
  • Python (requests)
import requests

url = "https://catalog.matatika.com:443/api/datacomponents/adcafd34-cc1b-4a78-abc6-81f4c7e55b6b"

headers = {
  'Authorization': ACCESS_TOKEN
}

response = requests.request("GET", url, headers=headers)

print(response.text.encode('utf8'))

Response

200 OK

Datacomponent with HAL links.

{
  "id" : "adcafd34-cc1b-4a78-abc6-81f4c7e55b6b",
  "created" : "2022-09-23T12:43:00.438046",
  "lastModified" : "2022-09-23T12:43:00.646195",
  "name" : "tap-matatika-sit",
  "properties" : { },
  "_embedded" : {
    "dataplugin" : {
      "id" : "fb2271ad-7a7c-4b8c-9dba-b3bcfada2aa2",
      "type" : "EXTRACTOR",
      "name" : "tap-matatika-sit",
      "namespace" : "tap_matatika_sit",
      "label" : "Matatika SIT",
      "description" : "Test extractor based on tap-spreadsheets-anywhere used during Matatika SIT runs",
      "logoUrl" : "/assets/images/datasource/tap-matatika-sit.svg",
      "hidden" : false,
      "documentationUrl" : "https://meltano.com/plugins/extractors/spreadsheets-anywhere.html",
      "pipUrl" : "git+https://github.com/ets/tap-spreadsheets-anywhere.git",
      "repositoryUrl" : "https://github.com/ets/tap-spreadsheets-anywhere",
      "executable" : "tap-spreadsheets-anywhere",
      "capabilities" : [ ],
      "select" : [ ],
      "update" : { },
      "vars" : { },
      "settings" : [ {
        "name" : "tables",
        "aliases" : [ ],
        "label" : "Tables",
        "value" : "[{\"path\":\"https://raw.githubusercontent.com/Matatika/matatika-examples/master/example_data\",\"name\":\"gitflixusers\",\"pattern\":\"GitFlixUsers.csv\",\"start_date\":\"2021-01-01T00:00:00Z\",\"key_properties\":[\"id\"],\"format\":\"csv\"}]",
        "kind" : "ARRAY",
        "description" : "An array holding json objects that each describe a set of targeted source files. See docs for details.",
        "protected" : false
      } ],
      "commands" : { },
      "matatikaHidden" : true,
      "_links" : {
        "self" : {
          "href" : "https://catalog.matatika.com/api/dataplugins/fb2271ad-7a7c-4b8c-9dba-b3bcfada2aa2"
        },
        "update dataplugin" : {
          "href" : "https://catalog.matatika.com/api/workspaces/2583dede-e0c3-45c0-8d5d-2e85f5de200a/dataplugins/fb2271ad-7a7c-4b8c-9dba-b3bcfada2aa2",
          "type" : "PUT"
        }
      }
    }
  },
  "managed" : false,
  "_links" : {
    "self" : {
      "href" : "https://catalog.matatika.com/api/datacomponents/adcafd34-cc1b-4a78-abc6-81f4c7e55b6b"
    },
    "update datacomponent" : {
      "href" : "https://catalog.matatika.com/api/datacomponents/adcafd34-cc1b-4a78-abc6-81f4c7e55b6b"
    },
    "delete datacomponent" : {
      "href" : "https://catalog.matatika.com/api/datacomponents/adcafd34-cc1b-4a78-abc6-81f4c7e55b6b"
    }
  }
}

Initialise a new datacomponent in a workspace

POST

/api/workspaces/{workspace-id}/datacomponents

Initialises a new datacomponent in the workspace {workspace-id}.

Request

Example Snippets

  • cURL
curl -H "Authorization: Bearer $ACCESS_TOKEN" 'https://catalog.matatika.com:443/api/workspaces/2583dede-e0c3-45c0-8d5d-2e85f5de200a/datacomponents' -i -X POST \
    -H 'Accept: application/json, application/javascript, text/javascript, text/json' \
    -H 'Content-Type: application/json'
  • Python (requests)
import requests

url = "https://catalog.matatika.com:443/api/workspaces/2583dede-e0c3-45c0-8d5d-2e85f5de200a/datacomponents"

headers = {
  'Authorization': ACCESS_TOKEN
}

response = requests.request("POST", url, headers=headers)

print(response.text.encode('utf8'))

Response

200 OK

datacomponent with HAL links.

{
  "id" : "adcafd34-cc1b-4a78-abc6-81f4c7e55b6b",
  "created" : "2022-09-23T12:43:00.292776988",
  "lastModified" : "2022-09-23T12:43:00.292777188",
  "properties" : { },
  "managed" : false,
  "_links" : {
    "self" : {
      "href" : "https://catalog.matatika.com/api/datacomponents/adcafd34-cc1b-4a78-abc6-81f4c7e55b6b"
    },
    "create datacomponent" : {
      "href" : "https://catalog.matatika.com/api/workspaces/2583dede-e0c3-45c0-8d5d-2e85f5de200a/datacomponents/adcafd34-cc1b-4a78-abc6-81f4c7e55b6b"
    }
  }
}

Create or update a datacomponent in a workspace

PUT

/api/workspaces/{workspace-id}/datacomponents/{datacomponent-id}

Creates or updates the datacomponent {datacomponent-id} in the workspace {workspace-id}.

Request

Body

Datacomponent resource.

{
  "name" : "tap-matatika-sit",
  "dataPlugin" : "tap-matatika-sit"
}

Example Snippets

  • cURL
curl -H "Authorization: Bearer $ACCESS_TOKEN" 'https://catalog.matatika.com:443/api/workspaces/2583dede-e0c3-45c0-8d5d-2e85f5de200a/datacomponents/adcafd34-cc1b-4a78-abc6-81f4c7e55b6b' -i -X PUT \
    -H 'Accept: application/json, application/javascript, text/javascript, text/json' \
    -H 'Content-Type: application/hal+json; charset=ISO-8859-1' \
    -d '{
  "name" : "tap-matatika-sit",
  "dataPlugin" : "tap-matatika-sit"
}'
  • Python (requests)
import requests

url = "https://catalog.matatika.com:443/api/workspaces/2583dede-e0c3-45c0-8d5d-2e85f5de200a/datacomponents/adcafd34-cc1b-4a78-abc6-81f4c7e55b6b"

data = {
  "name" : "tap-matatika-sit",
  "dataPlugin" : "tap-matatika-sit"
}
headers = {
  'Authorization': ACCESS_TOKEN
}

response = requests.request("PUT", url, headers=headers, data=data)

print(response.text.encode('utf8'))

Response

200 OK / 201 Created

Datacomponent with HAL links.

{
  "id" : "adcafd34-cc1b-4a78-abc6-81f4c7e55b6b",
  "created" : "2022-09-23T12:43:00.438046",
  "lastModified" : "2022-09-23T12:43:00.438046",
  "name" : "tap-matatika-sit",
  "properties" : { },
  "_embedded" : {
    "dataplugin" : {
      "id" : "fb2271ad-7a7c-4b8c-9dba-b3bcfada2aa2",
      "type" : "EXTRACTOR",
      "name" : "tap-matatika-sit",
      "namespace" : "tap_matatika_sit",
      "label" : "Matatika SIT",
      "description" : "Test extractor based on tap-spreadsheets-anywhere used during Matatika SIT runs",
      "logoUrl" : "/assets/images/datasource/tap-matatika-sit.svg",
      "hidden" : false,
      "documentationUrl" : "https://meltano.com/plugins/extractors/spreadsheets-anywhere.html",
      "pipUrl" : "git+https://github.com/ets/tap-spreadsheets-anywhere.git",
      "repositoryUrl" : "https://github.com/ets/tap-spreadsheets-anywhere",
      "executable" : "tap-spreadsheets-anywhere",
      "capabilities" : [ ],
      "select" : [ ],
      "update" : { },
      "vars" : { },
      "settings" : [ {
        "name" : "tables",
        "aliases" : [ ],
        "label" : "Tables",
        "value" : "[{\"path\":\"https://raw.githubusercontent.com/Matatika/matatika-examples/master/example_data\",\"name\":\"gitflixusers\",\"pattern\":\"GitFlixUsers.csv\",\"start_date\":\"2021-01-01T00:00:00Z\",\"key_properties\":[\"id\"],\"format\":\"csv\"}]",
        "kind" : "ARRAY",
        "description" : "An array holding json objects that each describe a set of targeted source files. See docs for details.",
        "protected" : false
      } ],
      "commands" : { },
      "matatikaHidden" : true,
      "_links" : {
        "self" : {
          "href" : "https://catalog.matatika.com/api/dataplugins/fb2271ad-7a7c-4b8c-9dba-b3bcfada2aa2"
        },
        "update dataplugin" : {
          "href" : "https://catalog.matatika.com/api/workspaces/2583dede-e0c3-45c0-8d5d-2e85f5de200a/dataplugins/fb2271ad-7a7c-4b8c-9dba-b3bcfada2aa2",
          "type" : "PUT"
        }
      }
    }
  },
  "managed" : false,
  "_links" : {
    "self" : {
      "href" : "https://catalog.matatika.com/api/datacomponents/adcafd34-cc1b-4a78-abc6-81f4c7e55b6b"
    },
    "update datacomponent" : {
      "href" : "https://catalog.matatika.com/api/datacomponents/adcafd34-cc1b-4a78-abc6-81f4c7e55b6b"
    },
    "delete datacomponent" : {
      "href" : "https://catalog.matatika.com/api/datacomponents/adcafd34-cc1b-4a78-abc6-81f4c7e55b6b"
    }
  }
}

Update a datacomponent

PUT

/api/datacomponents/{datacomponent-id}

Updates the datacomponent {datacomponent-id}.

Prerequisites

  • Datacomponent {datacomponent-id} must exist

Request

Body

Datacomponent resource.

{
  "name" : "tap-matatika-sit",
  "dataPlugin" : "tap-matatika-sit"
}

Example Snippets

  • cURL
curl -H "Authorization: Bearer $ACCESS_TOKEN" 'https://catalog.matatika.com:443/api/datacomponents/adcafd34-cc1b-4a78-abc6-81f4c7e55b6b' -i -X PUT \
    -H 'Accept: application/json, application/javascript, text/javascript, text/json' \
    -H 'Content-Type: application/hal+json; charset=ISO-8859-1' \
    -d '{
  "name" : "tap-matatika-sit",
  "dataPlugin" : "tap-matatika-sit"
}'
  • Python (requests)
import requests

url = "https://catalog.matatika.com:443/api/datacomponents/adcafd34-cc1b-4a78-abc6-81f4c7e55b6b"

data = {
  "name" : "tap-matatika-sit",
  "dataPlugin" : "tap-matatika-sit"
}
headers = {
  'Authorization': ACCESS_TOKEN
}

response = requests.request("PUT", url, headers=headers, data=data)

print(response.text.encode('utf8'))

Response

200 OK

Datacomponent with HAL links.

{
  "id" : "adcafd34-cc1b-4a78-abc6-81f4c7e55b6b",
  "created" : "2022-09-23T12:43:00.438046",
  "lastModified" : "2022-09-23T12:43:00.646195",
  "name" : "tap-matatika-sit",
  "properties" : { },
  "_embedded" : {
    "dataplugin" : {
      "id" : "fb2271ad-7a7c-4b8c-9dba-b3bcfada2aa2",
      "type" : "EXTRACTOR",
      "name" : "tap-matatika-sit",
      "namespace" : "tap_matatika_sit",
      "label" : "Matatika SIT",
      "description" : "Test extractor based on tap-spreadsheets-anywhere used during Matatika SIT runs",
      "logoUrl" : "/assets/images/datasource/tap-matatika-sit.svg",
      "hidden" : false,
      "documentationUrl" : "https://meltano.com/plugins/extractors/spreadsheets-anywhere.html",
      "pipUrl" : "git+https://github.com/ets/tap-spreadsheets-anywhere.git",
      "repositoryUrl" : "https://github.com/ets/tap-spreadsheets-anywhere",
      "executable" : "tap-spreadsheets-anywhere",
      "capabilities" : [ ],
      "select" : [ ],
      "update" : { },
      "vars" : { },
      "settings" : [ {
        "name" : "tables",
        "aliases" : [ ],
        "label" : "Tables",
        "value" : "[{\"path\":\"https://raw.githubusercontent.com/Matatika/matatika-examples/master/example_data\",\"name\":\"gitflixusers\",\"pattern\":\"GitFlixUsers.csv\",\"start_date\":\"2021-01-01T00:00:00Z\",\"key_properties\":[\"id\"],\"format\":\"csv\"}]",
        "kind" : "ARRAY",
        "description" : "An array holding json objects that each describe a set of targeted source files. See docs for details.",
        "protected" : false
      } ],
      "commands" : { },
      "matatikaHidden" : true,
      "_links" : {
        "self" : {
          "href" : "https://catalog.matatika.com/api/dataplugins/fb2271ad-7a7c-4b8c-9dba-b3bcfada2aa2"
        },
        "update dataplugin" : {
          "href" : "https://catalog.matatika.com/api/workspaces/2583dede-e0c3-45c0-8d5d-2e85f5de200a/dataplugins/fb2271ad-7a7c-4b8c-9dba-b3bcfada2aa2",
          "type" : "PUT"
        }
      }
    }
  },
  "managed" : false,
  "_links" : {
    "self" : {
      "href" : "https://catalog.matatika.com/api/datacomponents/adcafd34-cc1b-4a78-abc6-81f4c7e55b6b"
    },
    "update datacomponent" : {
      "href" : "https://catalog.matatika.com/api/datacomponents/adcafd34-cc1b-4a78-abc6-81f4c7e55b6b"
    },
    "delete datacomponent" : {
      "href" : "https://catalog.matatika.com/api/datacomponents/adcafd34-cc1b-4a78-abc6-81f4c7e55b6b"
    }
  }
}

Delete a datacomponent

DELETE

/api/datacomponents/{datacomponent-id}

Deletes the datacomponent {datacomponent-id}.

Request

Example Snippets

  • cURL
curl -H "Authorization: Bearer $ACCESS_TOKEN" 'https://catalog.matatika.com:443/api/datacomponents/adcafd34-cc1b-4a78-abc6-81f4c7e55b6b' -i -X DELETE \
    -H 'Accept: application/json, application/javascript, text/javascript, text/json' \
    -H 'Content-Type: application/json'
  • Python (requests)
import requests

url = "https://catalog.matatika.com:443/api/datacomponents/adcafd34-cc1b-4a78-abc6-81f4c7e55b6b"

headers = {
  'Authorization': ACCESS_TOKEN
}

response = requests.request("DELETE", url, headers=headers)

print(response.text.encode('utf8'))

Response

204 No Content

No response body provided.