.

extension-collector/pom.xml

@@ -0,0 +1,41 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+    <parent>
+        <groupId>io.github.jwdeveloper.tiktok</groupId>
+        <artifactId>TikTokLiveJava</artifactId>
+        <version>1.0.14-Release</version>
+    </parent>
+
+
+    <artifactId>extension-collector</artifactId>
+
+
+    <dependencies>
+        <dependency>
+            <groupId>io.github.jwdeveloper.tiktok</groupId>
+            <artifactId>API</artifactId>
+            <version>${project.version}</version>
+            <scope>compile</scope>
+        </dependency>
+        <dependency>
+            <groupId>com.google.protobuf</groupId>
+            <artifactId>protobuf-java</artifactId>
+            <version>3.24.1</version>
+        </dependency>
+        <dependency>
+            <groupId>org.mongodb</groupId>
+            <artifactId>mongodb-driver-sync</artifactId>
+            <version>4.4.0</version>
+        </dependency>
+        <dependency>
+            <groupId>io.github.jwdeveloper.tiktok</groupId>
+            <artifactId>API</artifactId>
+            <version>1.0.17-Release</version>
+            <scope>compile</scope>
+        </dependency>
+    </dependencies>
+
+</project>

extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/TikTokLiveCollector.java

@@ -0,0 +1,40 @@
+/*
+ * Copyright (c) 2023-2023 jwdeveloper jacekwoln@gmail.com
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining
+ * a copy of this software and associated documentation files (the
+ * "Software"), to deal in the Software without restriction, including
+ * without limitation the rights to use, copy, modify, merge, publish,
+ * distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to
+ * the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+ * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+ * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+ * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+package io.github.jwdeveloper.tiktok.extension.collector;
+
+import io.github.jwdeveloper.tiktok.extension.collector.api.LiveDataCollector;
+import io.github.jwdeveloper.tiktok.extension.collector.api.data.LiveDataCollectorSettings;
+import io.github.jwdeveloper.tiktok.extension.collector.impl.TikTokLiveDataCollector;
+
+import java.util.function.Consumer;
+
+public class TikTokLiveCollector
+{
+
+    public static TikTokLiveDataCollector use(Consumer<LiveDataCollectorSettings> consumer)
+    {
+        var settings = new LiveDataCollectorSettings();
+        consumer.accept(settings);
+        return new TikTokLiveDataCollector(settings);
+    }
+}

extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/api/LiveDataCollector.java

@@ -0,0 +1,30 @@
+/*
+ * Copyright (c) 2023-2023 jwdeveloper jacekwoln@gmail.com
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining
+ * a copy of this software and associated documentation files (the
+ * "Software"), to deal in the Software without restriction, including
+ * without limitation the rights to use, copy, modify, merge, publish,
+ * distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to
+ * the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+ * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+ * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+ * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+package io.github.jwdeveloper.tiktok.extension.collector.api;
+
+import io.github.jwdeveloper.tiktok.listener.TikTokEventListener;
+
+public interface LiveDataCollector extends TikTokEventListener
+{
+
+}

extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/api/data/CollectorListenerSettings.java

@@ -0,0 +1,13 @@
+package io.github.jwdeveloper.tiktok.extension.collector.api.data;
+
+import lombok.Data;
+import org.bson.Document;
+
+import java.util.Map;
+import java.util.function.Function;
+
+@Data
+public class CollectorListenerSettings {
+    private Map<String, Object> extraFields;
+    private Function<Document, Boolean> filter;
+}

extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/api/data/LiveDataCollectorSettings.java

@@ -0,0 +1,48 @@
+/*
+ * Copyright (c) 2023-2023 jwdeveloper jacekwoln@gmail.com
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining
+ * a copy of this software and associated documentation files (the
+ * "Software"), to deal in the Software without restriction, including
+ * without limitation the rights to use, copy, modify, merge, publish,
+ * distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to
+ * the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+ * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+ * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+ * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+package io.github.jwdeveloper.tiktok.extension.collector.api.data;
+
+import lombok.Data;
+
+import java.util.function.Consumer;
+
+@Data
+public class LiveDataCollectorSettings {
+
+    private String connectionUrl;
+
+    private String databaseName;
+
+    private String sessionTag;
+
+
+    public void setConnectionUrl(String connectionUrl) {
+        this.connectionUrl = connectionUrl;
+    }
+
+    public void setConnectionUrl(Consumer<MongoDBConnectionStringBuilder> consumer) {
+        var builder = new MongoDBConnectionStringBuilder();
+        consumer.accept(builder);
+        connectionUrl = builder.build();
+    }
+}

extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/api/data/MongoDBConnectionStringBuilder.java

@@ -0,0 +1,55 @@
+/*
+ * Copyright (c) 2023-2023 jwdeveloper jacekwoln@gmail.com
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining
+ * a copy of this software and associated documentation files (the
+ * "Software"), to deal in the Software without restriction, including
+ * without limitation the rights to use, copy, modify, merge, publish,
+ * distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to
+ * the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+ * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+ * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+ * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+package io.github.jwdeveloper.tiktok.extension.collector.api.data;
+
+public class MongoDBConnectionStringBuilder {
+    private String username;
+    private String password;
+    private String database;
+    private String cluster;
+
+    public MongoDBConnectionStringBuilder setUsername(String username) {
+        this.username = username;
+        return this;
+    }
+
+    public MongoDBConnectionStringBuilder setPassword(String password) {
+        this.password = password;
+        return this;
+    }
+
+    public MongoDBConnectionStringBuilder setDatabase(String database) {
+        this.database = database;
+        return this;
+    }
+
+    public MongoDBConnectionStringBuilder setCluster(String cluster) {
+        this.cluster = cluster;
+        return this;
+    }
+
+    public String build() {
+        return String.format("mongodb+srv://%s:%s@%s/%s?retryWrites=true&w=majority",
+                username, password, cluster, database);
+    }
+}

extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/impl/TikTokLiveDataCollector.java

@@ -0,0 +1,90 @@
+/*
+ * Copyright (c) 2023-2023 jwdeveloper jacekwoln@gmail.com
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining
+ * a copy of this software and associated documentation files (the
+ * "Software"), to deal in the Software without restriction, including
+ * without limitation the rights to use, copy, modify, merge, publish,
+ * distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to
+ * the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+ * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+ * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+ * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+package io.github.jwdeveloper.tiktok.extension.collector.impl;
+
+import com.mongodb.ConnectionString;
+import com.mongodb.MongoClientSettings;
+import com.mongodb.ServerApi;
+import com.mongodb.ServerApiVersion;
+import com.mongodb.client.MongoClient;
+import com.mongodb.client.MongoClients;
+import com.mongodb.client.MongoCollection;
+import com.mongodb.client.MongoDatabase;
+import com.mongodb.client.model.Indexes;
+import io.github.jwdeveloper.tiktok.extension.collector.api.data.CollectorListenerSettings;
+import io.github.jwdeveloper.tiktok.extension.collector.api.data.LiveDataCollectorSettings;
+import org.bson.Document;
+
+import java.util.Map;
+import java.util.TreeMap;
+import java.util.function.Function;
+
+public class TikTokLiveDataCollector {
+
+    private final LiveDataCollectorSettings settings;
+    private MongoClient mongoClient;
+    private MongoDatabase database;
+    private MongoCollection<Document> collection;
+
+    public TikTokLiveDataCollector(LiveDataCollectorSettings settings) {
+        this.settings = settings;
+    }
+
+
+    public void connectDatabase() {
+        var serverApi = ServerApi.builder()
+                .version(ServerApiVersion.V1)
+                .build();
+        var mongoSettings = MongoClientSettings.builder()
+                .applyConnectionString(new ConnectionString(settings.getConnectionUrl()))
+                .serverApi(serverApi)
+                .build();
+
+        mongoClient = MongoClients.create(mongoSettings);
+        database = mongoClient.getDatabase(settings.getDatabaseName());
+        collection = database.getCollection("data");
+        collection.createIndex(Indexes.hashed("session"));
+        collection.createIndex(Indexes.hashed("dataType"));
+    }
+
+
+    public void disconnectDatabase() {
+        mongoClient.close();
+    }
+
+    public TikTokLiveDataCollectorListener newListener() {
+        return newListener(Map.of());
+    }
+
+    public TikTokLiveDataCollectorListener newListener(Map<String, Object> additionalFields) {
+        return newListener(additionalFields, (e)->true);
+    }
+
+    public TikTokLiveDataCollectorListener newListener(Map<String, Object> additionalFields,
+                                                       Function<Document, Boolean> filter) {
+        var settings = new CollectorListenerSettings();
+        settings.setExtraFields(additionalFields);
+        settings.setFilter(filter);
+        return new TikTokLiveDataCollectorListener(collection, settings);
+    }
+}

extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/impl/TikTokLiveDataCollectorListener.java

@@ -0,0 +1,122 @@
+package io.github.jwdeveloper.tiktok.extension.collector.impl;
+
+import com.mongodb.client.MongoCollection;
+import io.github.jwdeveloper.tiktok.annotations.TikTokEventObserver;
+import io.github.jwdeveloper.tiktok.data.events.TikTokErrorEvent;
+import io.github.jwdeveloper.tiktok.data.events.common.TikTokEvent;
+import io.github.jwdeveloper.tiktok.data.events.control.TikTokConnectingEvent;
+import io.github.jwdeveloper.tiktok.data.events.websocket.TikTokWebsocketResponseEvent;
+import io.github.jwdeveloper.tiktok.exceptions.TikTokLiveMessageException;
+import io.github.jwdeveloper.tiktok.extension.collector.api.LiveDataCollector;
+import io.github.jwdeveloper.tiktok.extension.collector.api.data.CollectorListenerSettings;
+import io.github.jwdeveloper.tiktok.live.LiveClient;
+import io.github.jwdeveloper.tiktok.messages.webcast.WebcastResponse;
+import io.github.jwdeveloper.tiktok.utils.JsonUtil;
+import org.bson.Document;
+
+import java.io.PrintWriter;
+import java.io.StringWriter;
+import java.util.Base64;
+import java.util.UUID;
+
+public class TikTokLiveDataCollectorListener implements LiveDataCollector {
+
+    private final MongoCollection<Document> collection;
+    private final CollectorListenerSettings settings;
+    private String sessionId;
+    private String userName;
+
+    public TikTokLiveDataCollectorListener(MongoCollection<Document> collection, CollectorListenerSettings settings) {
+        this.collection = collection;
+        this.settings = settings;
+    }
+
+
+    @TikTokEventObserver
+    private void onResponse(LiveClient liveClient, TikTokWebsocketResponseEvent event) {
+        includeResponse(liveClient, event.getResponse());
+        event.getResponse().getMessagesList().forEach(message ->
+        {
+            includeMessage(liveClient, message);
+        });
+    }
+
+    @TikTokEventObserver
+    private void onEvent(LiveClient liveClient, TikTokEvent event) {
+        if (event instanceof TikTokConnectingEvent) {
+            sessionId = UUID.randomUUID().toString();
+            userName = liveClient.getRoomInfo().getHostName();
+        }
+
+        if (event instanceof TikTokErrorEvent) {
+            return;
+        }
+
+        includeEvent(event);
+    }
+
+    @TikTokEventObserver
+    private void onError(LiveClient liveClient, TikTokErrorEvent event) {
+        event.getException().printStackTrace();
+        includeError(event);
+    }
+
+
+    private void includeResponse(LiveClient liveClient, WebcastResponse message) {
+        var messageContent = Base64.getEncoder().encodeToString(message.toByteArray());
+        insertDocument(createDocument("response", "webcast", messageContent));
+    }
+
+    private void includeMessage(LiveClient liveClient, WebcastResponse.Message message) {
+        var method = message.getMethod();
+        var messageContent = Base64.getEncoder().encodeToString(message.getPayload().toByteArray());
+
+        insertDocument(createDocument("message", method, messageContent));
+    }
+
+    private void includeEvent(TikTokEvent event) {
+        var json = JsonUtil.toJson(event);
+        var content = Base64.getEncoder().encodeToString(json.getBytes());
+        var name = event.getClass().getSimpleName();
+        insertDocument(createDocument("event", name, content));
+    }
+
+    private void includeError(TikTokErrorEvent event) {
+        var exception = event.getException();
+        var exceptionName = event.getException().getClass().getSimpleName();
+
+        var sw = new StringWriter();
+        var pw = new PrintWriter(sw);
+        event.getException().printStackTrace(pw);
+        var content = sw.toString();
+
+        var doc = createDocument("error", exceptionName, content);
+        if (exception instanceof TikTokLiveMessageException ex) {
+            doc.append("message", ex.messageToBase64())
+                    .append("response", ex.webcastResponseToBase64());
+        }
+        insertDocument(doc);
+    }
+
+
+    private void insertDocument(Document document) {
+        if (!settings.getFilter().apply(document)) {
+            return;
+        }
+        collection.insertOne(document);
+    }
+
+
+    private Document createDocument(String dataType, String dataTypeName, String content) {
+        var doc = new Document();
+        doc.append("session", sessionId);
+        for (var entry : settings.getExtraFields().entrySet()) {
+            doc.append(entry.getKey(), entry.getValue());
+        }
+        doc.append("tiktokUser", userName);
+        doc.append("dataType", dataType);
+        doc.append("dataTypeName", dataTypeName);
+        doc.append("content", content);
+        return doc;
+    }
+}