Use the Kafka engine to consume data directly from Kafka compatible topics.
For more details on the Kafka engine, see the ClickHouse documentation on Kafka integration.
Kafka tables are streaming interfaces that don't persist data. Use a MaterializedView to continuously move data to a table.
import { OlapTable, ClickHouseEngines, MaterializedView, sql } from '@514labs/moose-lib'; interface KafkaEvent { eventId: string; userId: string; timestamp: number; // Unix seconds for JSONEachRow} // Kafka table - reads from topic, doesn't persistexport const kafkaSource = new OlapTable<KafkaEvent>("kafka_events", { engine: ClickHouseEngines.Kafka, brokerList: "redpanda:9092", topicList: "events", groupName: "my_consumer_group", format: "JSONEachRow", settings: { kafka_num_consumers: "1", }}); // MaterializedView moves data to MergeTree for persistenceconst cols = kafkaSource.columns;export const eventsMV = new MaterializedView<KafkaEvent>({ tableName: "events_dest", materializedViewName: "events_mv", orderByFields: ["eventId"], selectStatement: sql`SELECT ${cols.eventId}, ${cols.userId}, ${cols.timestamp} FROM ${kafkaSource}`, selectTables: [kafkaSource],});orderByFields