Working on collecting to files!

extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/TikTokLiveCollector.java

@@ -22,19 +22,25 @@
  */
 package io.github.jwdeveloper.tiktok.extension.collector;
 
-import io.github.jwdeveloper.tiktok.extension.collector.api.LiveDataCollector;
+import io.github.jwdeveloper.tiktok.extension.collector.api.file.FileDataCollectorSettings;
-import io.github.jwdeveloper.tiktok.extension.collector.api.data.LiveDataCollectorSettings;
+import io.github.jwdeveloper.tiktok.extension.collector.api.mongo.MongoDataCollectorSettings;
-import io.github.jwdeveloper.tiktok.extension.collector.impl.TikTokLiveDataCollector;
+import io.github.jwdeveloper.tiktok.extension.collector.impl.*;
 
 import java.util.function.Consumer;
 
 public class TikTokLiveCollector
 {
-
+    public static MongoDataCollector useMongo(Consumer<MongoDataCollectorSettings> consumer)
-    public static TikTokLiveDataCollector use(Consumer<LiveDataCollectorSettings> consumer)
     {
-        var settings = new LiveDataCollectorSettings();
+        var settings = new MongoDataCollectorSettings();
         consumer.accept(settings);
-        return new TikTokLiveDataCollector(settings);
+        return new MongoDataCollector(settings);
+    }
+
+    public static FileDataCollector useFile(Consumer<FileDataCollectorSettings> consumer)
+    {
+        var settings = new FileDataCollectorSettings();
+        consumer.accept(settings);
+        return new FileDataCollector(settings);
     }
 }

extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/api/data/CollectorListenerSettings.java

@@ -1,7 +1,6 @@
 package io.github.jwdeveloper.tiktok.extension.collector.api.data;
 
 import lombok.Data;
-import org.bson.Document;
 
 import java.util.Map;
 import java.util.function.Function;
@@ -9,5 +8,5 @@ import java.util.function.Function;
 @Data
 public class CollectorListenerSettings {
     private Map<String, Object> extraFields;
-    private Function<Document, Boolean> filter;
+    private Function<Object, Boolean> filter;
 }

extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/api/file/FileDataCollectorSettings.java

@@ -0,0 +1,33 @@
+/*
+ * Copyright (c) 2023-2023 jwdeveloper jacekwoln@gmail.com
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining
+ * a copy of this software and associated documentation files (the
+ * "Software"), to deal in the Software without restriction, including
+ * without limitation the rights to use, copy, modify, merge, publish,
+ * distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to
+ * the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+ * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+ * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+ * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+package io.github.jwdeveloper.tiktok.extension.collector.api.file;
+
+import lombok.Data;
+
+import java.io.File;
+
+@Data
+public class FileDataCollectorSettings {
+
+    private File parentFile;
+}

extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/api/mongo/MongoDBConnectionStringBuilder.java

@@ -20,34 +20,19 @@
  * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
  * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  */
-package io.github.jwdeveloper.tiktok.extension.collector.api.data;
+package io.github.jwdeveloper.tiktok.extension.collector.api.mongo;
 
+import lombok.Setter;
+import lombok.experimental.Accessors;
+
+@Setter
+@Accessors(chain = true)
 public class MongoDBConnectionStringBuilder {
     private String username;
     private String password;
     private String database;
     private String cluster;
 
-    public MongoDBConnectionStringBuilder setUsername(String username) {
-        this.username = username;
-        return this;
-    }
-
-    public MongoDBConnectionStringBuilder setPassword(String password) {
-        this.password = password;
-        return this;
-    }
-
-    public MongoDBConnectionStringBuilder setDatabase(String database) {
-        this.database = database;
-        return this;
-    }
-
-    public MongoDBConnectionStringBuilder setCluster(String cluster) {
-        this.cluster = cluster;
-        return this;
-    }
-
     public String build() {
         return String.format("mongodb+srv://%s:%s@%s/%s?retryWrites=true&w=majority",
                 username, password, cluster, database);

extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/api/mongo/MongoDataCollectorSettings.java

@@ -20,14 +20,14 @@
  * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
  * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  */
-package io.github.jwdeveloper.tiktok.extension.collector.api.data;
+package io.github.jwdeveloper.tiktok.extension.collector.api.mongo;
 
-import lombok.Data;
+import lombok.*;
 
 import java.util.function.Consumer;
 
 @Data
-public class LiveDataCollectorSettings {
+public class MongoDataCollectorSettings {
 
     private String connectionUrl;
 
@@ -35,11 +35,6 @@ public class LiveDataCollectorSettings {
 
     private String sessionTag;
 
-
-    public void setConnectionUrl(String connectionUrl) {
-        this.connectionUrl = connectionUrl;
-    }
-
     public void setConnectionUrl(Consumer<MongoDBConnectionStringBuilder> consumer) {
         var builder = new MongoDBConnectionStringBuilder();
         consumer.accept(builder);

extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/impl/FileDataCollector.java

@@ -0,0 +1,54 @@
+/*
+ * Copyright (c) 2023-2023 jwdeveloper jacekwoln@gmail.com
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining
+ * a copy of this software and associated documentation files (the
+ * "Software"), to deal in the Software without restriction, including
+ * without limitation the rights to use, copy, modify, merge, publish,
+ * distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to
+ * the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+ * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+ * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+ * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+package io.github.jwdeveloper.tiktok.extension.collector.impl;
+
+import io.github.jwdeveloper.tiktok.extension.collector.api.data.CollectorListenerSettings;
+import io.github.jwdeveloper.tiktok.extension.collector.api.file.FileDataCollectorSettings;
+
+import java.util.Map;
+import java.util.function.Function;
+
+public class FileDataCollector {
+
+    private final FileDataCollectorSettings settings;
+
+    public FileDataCollector(FileDataCollectorSettings settings) {
+        this.settings = settings;
+    }
+
+    public FileDataCollectorListener newListener() {
+        return newListener(Map.of());
+    }
+
+    public FileDataCollectorListener newListener(Map<String, Object> additionalFields) {
+        return newListener(additionalFields, (e)->true);
+    }
+
+    public FileDataCollectorListener newListener(Map<String, Object> additionalFields,
+                                                  Function<Object, Boolean> filter) {
+        var settings = new CollectorListenerSettings();
+        settings.setExtraFields(additionalFields);
+        settings.setFilter(filter);
+        return new FileDataCollectorListener(this.settings, settings);
+    }
+}

extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/impl/FileDataCollectorListener.java

@@ -0,0 +1,124 @@
+package io.github.jwdeveloper.tiktok.extension.collector.impl;
+
+import com.google.gson.*;
+import io.github.jwdeveloper.tiktok.annotations.TikTokEventObserver;
+import io.github.jwdeveloper.tiktok.data.events.TikTokErrorEvent;
+import io.github.jwdeveloper.tiktok.data.events.common.TikTokEvent;
+import io.github.jwdeveloper.tiktok.data.events.control.TikTokConnectingEvent;
+import io.github.jwdeveloper.tiktok.data.events.websocket.TikTokWebsocketResponseEvent;
+import io.github.jwdeveloper.tiktok.exceptions.TikTokLiveMessageException;
+import io.github.jwdeveloper.tiktok.extension.collector.api.LiveDataCollector;
+import io.github.jwdeveloper.tiktok.extension.collector.api.data.CollectorListenerSettings;
+import io.github.jwdeveloper.tiktok.extension.collector.api.file.FileDataCollectorSettings;
+import io.github.jwdeveloper.tiktok.live.LiveClient;
+import io.github.jwdeveloper.tiktok.messages.webcast.WebcastResponse;
+import io.github.jwdeveloper.tiktok.utils.JsonUtil;
+
+import java.io.*;
+import java.nio.file.*;
+import java.util.*;
+
+public class FileDataCollectorListener implements LiveDataCollector {
+
+    private final FileDataCollectorSettings fileSettings;
+    private final CollectorListenerSettings collectorSettings;
+    private String sessionId;
+    private String userName;
+
+    public FileDataCollectorListener(FileDataCollectorSettings fileSettings, CollectorListenerSettings collectorSettings) {
+        this.fileSettings = fileSettings;
+        this.collectorSettings = collectorSettings;
+    }
+
+    @TikTokEventObserver
+    private void onResponse(LiveClient liveClient, TikTokWebsocketResponseEvent event) {
+        includeResponse(liveClient, event.getResponse());
+        event.getResponse().getMessagesList().forEach(message -> includeMessage(liveClient, message));
+    }
+
+    @TikTokEventObserver
+    private void onEvent(LiveClient liveClient, TikTokEvent event) {
+        if (event instanceof TikTokConnectingEvent) {
+            sessionId = UUID.randomUUID().toString();
+            userName = liveClient.getRoomInfo().getHostName();
+        }
+
+        if (event instanceof TikTokErrorEvent) {
+            return;
+        }
+
+        includeEvent(event);
+    }
+
+    @TikTokEventObserver
+    private void onError(LiveClient liveClient, TikTokErrorEvent event) {
+        event.getException().printStackTrace();
+        includeError(event);
+    }
+
+
+    private void includeResponse(LiveClient liveClient, WebcastResponse message) {
+        var messageContent = Base64.getEncoder().encodeToString(message.toByteArray());
+        saveJson(createJson("response", "webcast", messageContent));
+    }
+
+    private void includeMessage(LiveClient liveClient, WebcastResponse.Message message) {
+        var method = message.getMethod();
+        var messageContent = Base64.getEncoder().encodeToString(message.getPayload().toByteArray());
+
+        saveJson(createJson("message", method, messageContent));
+    }
+
+    private void includeEvent(TikTokEvent event) {
+        var json = JsonUtil.toJson(event);
+        var content = Base64.getEncoder().encodeToString(json.getBytes());
+        var name = event.getClass().getSimpleName();
+        saveJson(createJson("event", name, content));
+    }
+
+    private void includeError(TikTokErrorEvent event) {
+        var exception = event.getException();
+        var exceptionName = event.getException().getClass().getSimpleName();
+
+        var sw = new StringWriter();
+        var pw = new PrintWriter(sw);
+        event.getException().printStackTrace(pw);
+        var content = sw.toString();
+
+        var json = createJson("error", exceptionName, content);
+        if (exception instanceof TikTokLiveMessageException ex) {
+            json.addProperty("message", ex.messageToBase64());
+            json.addProperty("response", ex.webcastResponseToBase64());
+        }
+        saveJson(json);
+    }
+
+    private void saveJson(JsonObject jsonObject) {
+        if (!collectorSettings.getFilter().apply(jsonObject)) {
+            return;
+        }
+        try {
+            File file = new File(fileSettings.getParentFile(), jsonObject.get("dataType").getAsString()+":"+jsonObject.get("dataTypeName").getAsString()+".txt");
+            file.createNewFile();
+            Files.writeString(file.toPath(), jsonObject.toString(), StandardOpenOption.APPEND);
+        } catch (IOException e) {
+            e.printStackTrace();
+        }
+    }
+
+    private JsonObject createJson(String dataType, String dataTypeName, String content) {
+        JsonObject data = new JsonObject();
+        data.addProperty("session", sessionId);
+        for (var entry : collectorSettings.getExtraFields().entrySet()) {
+            if (entry.getValue() instanceof JsonElement element)
+                data.add(entry.getKey(), element);
+            else
+                data.addProperty(entry.getKey(), entry.getValue().toString());
+        }
+        data.addProperty("tiktokUser", userName);
+        data.addProperty("dataType", dataType);
+        data.addProperty("dataTypeName", dataTypeName);
+        data.addProperty("content", content);
+        return data;
+    }
+}

extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/impl/MongoDataCollector.java

@@ -32,25 +32,23 @@ import com.mongodb.client.MongoCollection;
 import com.mongodb.client.MongoDatabase;
 import com.mongodb.client.model.Indexes;
 import io.github.jwdeveloper.tiktok.extension.collector.api.data.CollectorListenerSettings;
-import io.github.jwdeveloper.tiktok.extension.collector.api.data.LiveDataCollectorSettings;
+import io.github.jwdeveloper.tiktok.extension.collector.api.mongo.MongoDataCollectorSettings;
 import org.bson.Document;
 
 import java.util.Map;
-import java.util.TreeMap;
 import java.util.function.Function;
 
-public class TikTokLiveDataCollector {
+public class MongoDataCollector {
 
-    private final LiveDataCollectorSettings settings;
+    private final MongoDataCollectorSettings settings;
     private MongoClient mongoClient;
     private MongoDatabase database;
     private MongoCollection<Document> collection;
 
-    public TikTokLiveDataCollector(LiveDataCollectorSettings settings) {
+    public MongoDataCollector(MongoDataCollectorSettings settings) {
         this.settings = settings;
     }
 
-
     public void connectDatabase() {
         var serverApi = ServerApi.builder()
                 .version(ServerApiVersion.V1)
@@ -72,19 +70,19 @@ public class TikTokLiveDataCollector {
         mongoClient.close();
     }
 
-    public TikTokLiveDataCollectorListener newListener() {
+    public MongoDataCollectorListener newListener() {
         return newListener(Map.of());
     }
 
-    public TikTokLiveDataCollectorListener newListener(Map<String, Object> additionalFields) {
+    public MongoDataCollectorListener newListener(Map<String, Object> additionalFields) {
         return newListener(additionalFields, (e)->true);
     }
 
-    public TikTokLiveDataCollectorListener newListener(Map<String, Object> additionalFields,
+    public MongoDataCollectorListener newListener(Map<String, Object> additionalFields,
-                                                       Function<Document, Boolean> filter) {
+                                                  Function<Object, Boolean> filter) {
         var settings = new CollectorListenerSettings();
         settings.setExtraFields(additionalFields);
         settings.setFilter(filter);
-        return new TikTokLiveDataCollectorListener(collection, settings);
+        return new MongoDataCollectorListener(collection, settings);
     }
 }

extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/impl/MongoDataCollectorListener.java

@@ -19,14 +19,14 @@ import java.io.StringWriter;
 import java.util.Base64;
 import java.util.UUID;
 
-public class TikTokLiveDataCollectorListener implements LiveDataCollector {
+public class MongoDataCollectorListener implements LiveDataCollector {
 
     private final MongoCollection<Document> collection;
     private final CollectorListenerSettings settings;
     private String sessionId;
     private String userName;
 
-    public TikTokLiveDataCollectorListener(MongoCollection<Document> collection, CollectorListenerSettings settings) {
+    public MongoDataCollectorListener(MongoCollection<Document> collection, CollectorListenerSettings settings) {
         this.collection = collection;
         this.settings = settings;
     }
@@ -35,10 +35,7 @@ public class TikTokLiveDataCollectorListener implements LiveDataCollector {
     @TikTokEventObserver
     private void onResponse(LiveClient liveClient, TikTokWebsocketResponseEvent event) {
         includeResponse(liveClient, event.getResponse());
-        event.getResponse().getMessagesList().forEach(message ->
+        event.getResponse().getMessagesList().forEach(message -> includeMessage(liveClient, message));
-        {
-            includeMessage(liveClient, message);
-        });
     }
 
     @TikTokEventObserver