Ingest Sample Data

Ingest Sample Data

Test out the /ingest/RawStarEvent API endpoint on some sample data with this curl command:

Terminal
curl -X POST http://localhost:4000/ingest/RawStarEvent \
     -H "Content-Type: application/json" \
     -d '{
  "action": "TEST-PAYLOAD",
  "starred_at": "2024-07-30T01:04:06Z",
  "repository": {
    "id": 668493044,
    "node_id": "R_kgDOJ9hk9A",
    "name": "moose",
    "full_name": "514-labs/moose",
    "private": false,
    "owner": {
      "login": "514-labs",
      "id": 140028474,
      "node_id": "O_kgDOCFiqOg",
      "avatar_url": "https://avatars.githubusercontent.com/u/140028474?v=4",
      "gravatar_id": "",
      "url": "https://api.github.com/users/514-labs",
      "html_url": "https://github.com/514-labs",
      "followers_url": "https://api.github.com/users/514-labs/followers",
      "following_url": "https://api.github.com/users/514-labs/following{/other_user}",
      "gists_url": "https://api.github.com/users/514-labs/gists{/gist_id}",
      "starred_url": "https://api.github.com/users/514-labs/starred{/owner}{/repo}",
      "subscriptions_url": "https://api.github.com/users/514-labs/subscriptions",
      "organizations_url": "https://api.github.com/users/514-labs/orgs",
      "repos_url": "https://api.github.com/users/514-labs/repos",
      "events_url": "https://api.github.com/users/514-labs/events{/privacy}",
      "received_events_url": "https://api.github.com/users/514-labs/received_events",
      "type": "Organization",
      "site_admin": false
    },
    "html_url": "https://github.com/514-labs/moose",
    "description": "The developer framework for your data & analytics stack",
    "fork": false,
    "url": "https://api.github.com/repos/514-labs/moose",
    "forks_url": "https://api.github.com/repos/514-labs/moose/forks",
    "keys_url": "https://api.github.com/repos/514-labs/moose/keys{/key_id}",
    "collaborators_url": "https://api.github.com/repos/514-labs/moose/collaborators{/collaborator}",
    "teams_url": "https://api.github.com/repos/514-labs/moose/teams",
    "hooks_url": "https://api.github.com/repos/514-labs/moose/hooks",
    "issue_events_url": "https://api.github.com/repos/514-labs/moose/issues/events{/number}",
    "events_url": "https://api.github.com/repos/514-labs/moose/events",
    "assignees_url": "https://api.github.com/repos/514-labs/moose/assignees{/user}",
    "branches_url": "https://api.github.com/repos/514-labs/moose/branches{/branch}",
    "tags_url": "https://api.github.com/repos/514-labs/moose/tags",
    "blobs_url": "https://api.github.com/repos/514-labs/moose/git/blobs{/sha}",
    "git_tags_url": "https://api.github.com/repos/514-labs/moose/git/tags{/sha}",
    "git_refs_url": "https://api.github.com/repos/514-labs/moose/git/refs{/sha}",
    "trees_url": "https://api.github.com/repos/514-labs/moose/git/trees{/sha}",
    "statuses_url": "https://api.github.com/repos/514-labs/moose/statuses/{sha}",
    "languages_url": "https://api.github.com/repos/514-labs/moose/languages",
    "stargazers_url": "https://api.github.com/repos/514-labs/moose/stargazers",
    "contributors_url": "https://api.github.com/repos/514-labs/moose/contributors",
    "subscribers_url": "https://api.github.com/repos/514-labs/moose/subscribers",
    "subscription_url": "https://api.github.com/repos/514-labs/moose/subscription",
    "commits_url": "https://api.github.com/repos/514-labs/moose/commits{/sha}",
    "git_commits_url": "https://api.github.com/repos/514-labs/moose/git/commits{/sha}",
    "comments_url": "https://api.github.com/repos/514-labs/moose/comments{/number}",
    "issue_comment_url": "https://api.github.com/repos/514-labs/moose/issues/comments{/number}",
    "contents_url": "https://api.github.com/repos/514-labs/moose/contents/{+path}",
    "compare_url": "https://api.github.com/repos/514-labs/moose/compare/{base}...{head}",
    "merges_url": "https://api.github.com/repos/514-labs/moose/merges",
    "archive_url": "https://api.github.com/repos/514-labs/moose/{archive_format}{/ref}",
    "downloads_url": "https://api.github.com/repos/514-labs/moose/downloads",
    "issues_url": "https://api.github.com/repos/514-labs/moose/issues{/number}",
    "pulls_url": "https://api.github.com/repos/514-labs/moose/pulls{/number}",
    "milestones_url": "https://api.github.com/repos/514-labs/moose/milestones{/number}",
    "notifications_url": "https://api.github.com/repos/514-labs/moose/notifications{?since,all,participating}",
    "labels_url": "https://api.github.com/repos/514-labs/moose/labels{/name}",
    "releases_url": "https://api.github.com/repos/514-labs/moose/releases{/id}",
    "deployments_url": "https://api.github.com/repos/514-labs/moose/deployments",
    "created_at": "2023-07-20T00:23:21Z",
    "updated_at": "2024-07-30T01:04:07Z",
    "pushed_at": "2024-07-30T00:32:38Z",
    "git_url": "git://github.com/514-labs/moose.git",
    "ssh_url": "git@github.com:514-labs/moose.git",
    "clone_url": "https://github.com/514-labs/moose.git",
    "svn_url": "https://github.com/514-labs/moose",
    "homepage": "https://www.moosejs.com",
    "size": 75529,
    "stargazers_count": 28,
    "watchers_count": 28,
    "language": "Rust",
    "has_issues": true,
    "has_projects": true,
    "has_downloads": true,
    "has_wiki": false,
    "has_pages": false,
    "has_discussions": true,
    "forks_count": 6,
    "mirror_url": "https://github.com/514-labs/moose",
    "archived": false,
    "disabled": false,
    "open_issues_count": 351,
    "license": {
      "key": "mit",
      "name": "MIT License",
      "spdx_id": "MIT",
      "url": "https://api.github.com/licenses/mit",
      "node_id": "MDc6TGljZW5zZTEz"
    },
    "allow_forking": true,
    "is_template": false,
    "web_commit_signoff_required": false,
    "topics": [
      "analytics",
      "data",
      "dataengineering",
      "deployment",
      "framework",
      "insights",
      "metrics",
      "python",
      "rust",
      "typescript"
    ],
    "visibility": "public",
    "forks": 6,
    "open_issues": 351,
    "watchers": 28,
    "default_branch": "main"
  },
  "organization": {
    "login": "514-labs",
    "id": 140028474,
    "node_id": "O_kgDOCFiqOg",
    "url": "https://api.github.com/orgs/514-labs",
    "repos_url": "https://api.github.com/orgs/514-labs/repos",
    "events_url": "https://api.github.com/orgs/514-labs/events",
    "hooks_url": "https://api.github.com/orgs/514-labs/hooks",
    "issues_url": "https://api.github.com/orgs/514-labs/issues",
    "members_url": "https://api.github.com/orgs/514-labs/members{/member}",
    "public_members_url": "https://api.github.com/orgs/514-labs/public_members{/member}",
    "avatar_url": "https://avatars.githubusercontent.com/u/140028474?v=4",
    "description": "build and deploy data-intensive apps with ease"
  },
  "sender": {
    "login": "armanjindal",
    "id": 58370547,
    "node_id": "MDQ6VXNlcjU4MzcwNTQ3",
    "avatar_url": "https://avatars.githubusercontent.com/u/58370547?v=4",
    "gravatar_id": "",
    "url": "https://api.github.com/users/armanjindal",
    "html_url": "https://github.com/armanjindal",
    "followers_url": "https://api.github.com/users/armanjindal/followers",
    "following_url": "https://api.github.com/users/armanjindal/following{/other_user}",
    "gists_url": "https://api.github.com/users/armanjindal/gists{/gist_id}",
    "starred_url": "https://api.github.com/users/armanjindal/starred{/owner}{/repo}",
    "subscriptions_url": "https://api.github.com/users/armanjindal/subscriptions",
    "organizations_url": "https://api.github.com/users/armanjindal/orgs",
    "repos_url": "https://api.github.com/users/armanjindal/repos",
    "events_url": "https://api.github.com/users/armanjindal/events{/privacy}",
    "received_events_url": "https://api.github.com/users/armanjindal/received_events",
    "type": "User",
    "site_admin": false
  }
}'

In the terminal window where you started your local dev server, you should see the following messages printed to the console:

POST ingest/RawStarEvent/0.0
SUCCESS /ingest/RawStarEvent

Query the RawStarEvent Table

In your database explorer, execute the following query:

SELECT * FROM local.RawStarEvent_0_0

Setting Up Your Database Explorer

VSCode SQL Tools Extension Setup
  • Click the SQLTools database icon in your extensions tab.
  • You should see the Clickhouse icon followed by the connection details for the database running on your local dev server: panda@localhost:18123/local.
  • To create and run a new query, click the New SQL File icon.

SQL Tools Setup