From 3ae73072ffd768314bcc20750f9bafc2a499f1ba Mon Sep 17 00:00:00 2001 From: kohlerpop1 Date: Wed, 28 Feb 2024 12:24:34 -0500 Subject: [PATCH] Working on collecting to files! --- .../collector/TikTokLiveCollector.java | 22 ++-- .../api/data/CollectorListenerSettings.java | 5 +- .../api/file/FileDataCollectorSettings.java | 33 +++++ .../MongoDBConnectionStringBuilder.java | 29 +--- .../MongoDataCollectorSettings.java} | 13 +- .../collector/impl/FileDataCollector.java | 54 ++++++++ .../impl/FileDataCollectorListener.java | 124 ++++++++++++++++++ ...Collector.java => MongoDataCollector.java} | 22 ++-- ...r.java => MongoDataCollectorListener.java} | 11 +- 9 files changed, 252 insertions(+), 61 deletions(-) create mode 100644 extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/api/file/FileDataCollectorSettings.java rename extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/api/{data => mongo}/MongoDBConnectionStringBuilder.java (71%) rename extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/api/{data/LiveDataCollectorSettings.java => mongo/MongoDataCollectorSettings.java} (86%) create mode 100644 extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/impl/FileDataCollector.java create mode 100644 extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/impl/FileDataCollectorListener.java rename extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/impl/{TikTokLiveDataCollector.java => MongoDataCollector.java} (80%) rename extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/impl/{TikTokLiveDataCollectorListener.java => MongoDataCollectorListener.java} (94%) diff --git a/extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/TikTokLiveCollector.java b/extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/TikTokLiveCollector.java index 0ba1894..8b30f42 100644 --- a/extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/TikTokLiveCollector.java +++ b/extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/TikTokLiveCollector.java @@ -22,19 +22,25 @@ */ package io.github.jwdeveloper.tiktok.extension.collector; -import io.github.jwdeveloper.tiktok.extension.collector.api.LiveDataCollector; -import io.github.jwdeveloper.tiktok.extension.collector.api.data.LiveDataCollectorSettings; -import io.github.jwdeveloper.tiktok.extension.collector.impl.TikTokLiveDataCollector; +import io.github.jwdeveloper.tiktok.extension.collector.api.file.FileDataCollectorSettings; +import io.github.jwdeveloper.tiktok.extension.collector.api.mongo.MongoDataCollectorSettings; +import io.github.jwdeveloper.tiktok.extension.collector.impl.*; import java.util.function.Consumer; public class TikTokLiveCollector { - - public static TikTokLiveDataCollector use(Consumer consumer) + public static MongoDataCollector useMongo(Consumer consumer) { - var settings = new LiveDataCollectorSettings(); + var settings = new MongoDataCollectorSettings(); consumer.accept(settings); - return new TikTokLiveDataCollector(settings); + return new MongoDataCollector(settings); } -} + + public static FileDataCollector useFile(Consumer consumer) + { + var settings = new FileDataCollectorSettings(); + consumer.accept(settings); + return new FileDataCollector(settings); + } +} \ No newline at end of file diff --git a/extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/api/data/CollectorListenerSettings.java b/extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/api/data/CollectorListenerSettings.java index 8399a05..7c4710c 100644 --- a/extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/api/data/CollectorListenerSettings.java +++ b/extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/api/data/CollectorListenerSettings.java @@ -1,7 +1,6 @@ package io.github.jwdeveloper.tiktok.extension.collector.api.data; import lombok.Data; -import org.bson.Document; import java.util.Map; import java.util.function.Function; @@ -9,5 +8,5 @@ import java.util.function.Function; @Data public class CollectorListenerSettings { private Map extraFields; - private Function filter; -} + private Function filter; +} \ No newline at end of file diff --git a/extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/api/file/FileDataCollectorSettings.java b/extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/api/file/FileDataCollectorSettings.java new file mode 100644 index 0000000..395b8fe --- /dev/null +++ b/extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/api/file/FileDataCollectorSettings.java @@ -0,0 +1,33 @@ +/* + * Copyright (c) 2023-2023 jwdeveloper jacekwoln@gmail.com + * + * Permission is hereby granted, free of charge, to any person obtaining + * a copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE + * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION + * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package io.github.jwdeveloper.tiktok.extension.collector.api.file; + +import lombok.Data; + +import java.io.File; + +@Data +public class FileDataCollectorSettings { + + private File parentFile; +} \ No newline at end of file diff --git a/extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/api/data/MongoDBConnectionStringBuilder.java b/extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/api/mongo/MongoDBConnectionStringBuilder.java similarity index 71% rename from extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/api/data/MongoDBConnectionStringBuilder.java rename to extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/api/mongo/MongoDBConnectionStringBuilder.java index 556a0cd..e1b4b85 100644 --- a/extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/api/data/MongoDBConnectionStringBuilder.java +++ b/extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/api/mongo/MongoDBConnectionStringBuilder.java @@ -20,36 +20,21 @@ * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package io.github.jwdeveloper.tiktok.extension.collector.api.data; +package io.github.jwdeveloper.tiktok.extension.collector.api.mongo; +import lombok.Setter; +import lombok.experimental.Accessors; + +@Setter +@Accessors(chain = true) public class MongoDBConnectionStringBuilder { private String username; private String password; private String database; private String cluster; - public MongoDBConnectionStringBuilder setUsername(String username) { - this.username = username; - return this; - } - - public MongoDBConnectionStringBuilder setPassword(String password) { - this.password = password; - return this; - } - - public MongoDBConnectionStringBuilder setDatabase(String database) { - this.database = database; - return this; - } - - public MongoDBConnectionStringBuilder setCluster(String cluster) { - this.cluster = cluster; - return this; - } - public String build() { return String.format("mongodb+srv://%s:%s@%s/%s?retryWrites=true&w=majority", username, password, cluster, database); } -} +} \ No newline at end of file diff --git a/extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/api/data/LiveDataCollectorSettings.java b/extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/api/mongo/MongoDataCollectorSettings.java similarity index 86% rename from extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/api/data/LiveDataCollectorSettings.java rename to extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/api/mongo/MongoDataCollectorSettings.java index 4597c5c..bdabb40 100644 --- a/extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/api/data/LiveDataCollectorSettings.java +++ b/extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/api/mongo/MongoDataCollectorSettings.java @@ -20,14 +20,14 @@ * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package io.github.jwdeveloper.tiktok.extension.collector.api.data; +package io.github.jwdeveloper.tiktok.extension.collector.api.mongo; -import lombok.Data; +import lombok.*; import java.util.function.Consumer; @Data -public class LiveDataCollectorSettings { +public class MongoDataCollectorSettings { private String connectionUrl; @@ -35,14 +35,9 @@ public class LiveDataCollectorSettings { private String sessionTag; - - public void setConnectionUrl(String connectionUrl) { - this.connectionUrl = connectionUrl; - } - public void setConnectionUrl(Consumer consumer) { var builder = new MongoDBConnectionStringBuilder(); consumer.accept(builder); connectionUrl = builder.build(); } -} +} \ No newline at end of file diff --git a/extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/impl/FileDataCollector.java b/extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/impl/FileDataCollector.java new file mode 100644 index 0000000..4ea9c1d --- /dev/null +++ b/extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/impl/FileDataCollector.java @@ -0,0 +1,54 @@ +/* + * Copyright (c) 2023-2023 jwdeveloper jacekwoln@gmail.com + * + * Permission is hereby granted, free of charge, to any person obtaining + * a copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE + * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION + * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package io.github.jwdeveloper.tiktok.extension.collector.impl; + +import io.github.jwdeveloper.tiktok.extension.collector.api.data.CollectorListenerSettings; +import io.github.jwdeveloper.tiktok.extension.collector.api.file.FileDataCollectorSettings; + +import java.util.Map; +import java.util.function.Function; + +public class FileDataCollector { + + private final FileDataCollectorSettings settings; + + public FileDataCollector(FileDataCollectorSettings settings) { + this.settings = settings; + } + + public FileDataCollectorListener newListener() { + return newListener(Map.of()); + } + + public FileDataCollectorListener newListener(Map additionalFields) { + return newListener(additionalFields, (e)->true); + } + + public FileDataCollectorListener newListener(Map additionalFields, + Function filter) { + var settings = new CollectorListenerSettings(); + settings.setExtraFields(additionalFields); + settings.setFilter(filter); + return new FileDataCollectorListener(this.settings, settings); + } +} \ No newline at end of file diff --git a/extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/impl/FileDataCollectorListener.java b/extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/impl/FileDataCollectorListener.java new file mode 100644 index 0000000..e724285 --- /dev/null +++ b/extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/impl/FileDataCollectorListener.java @@ -0,0 +1,124 @@ +package io.github.jwdeveloper.tiktok.extension.collector.impl; + +import com.google.gson.*; +import io.github.jwdeveloper.tiktok.annotations.TikTokEventObserver; +import io.github.jwdeveloper.tiktok.data.events.TikTokErrorEvent; +import io.github.jwdeveloper.tiktok.data.events.common.TikTokEvent; +import io.github.jwdeveloper.tiktok.data.events.control.TikTokConnectingEvent; +import io.github.jwdeveloper.tiktok.data.events.websocket.TikTokWebsocketResponseEvent; +import io.github.jwdeveloper.tiktok.exceptions.TikTokLiveMessageException; +import io.github.jwdeveloper.tiktok.extension.collector.api.LiveDataCollector; +import io.github.jwdeveloper.tiktok.extension.collector.api.data.CollectorListenerSettings; +import io.github.jwdeveloper.tiktok.extension.collector.api.file.FileDataCollectorSettings; +import io.github.jwdeveloper.tiktok.live.LiveClient; +import io.github.jwdeveloper.tiktok.messages.webcast.WebcastResponse; +import io.github.jwdeveloper.tiktok.utils.JsonUtil; + +import java.io.*; +import java.nio.file.*; +import java.util.*; + +public class FileDataCollectorListener implements LiveDataCollector { + + private final FileDataCollectorSettings fileSettings; + private final CollectorListenerSettings collectorSettings; + private String sessionId; + private String userName; + + public FileDataCollectorListener(FileDataCollectorSettings fileSettings, CollectorListenerSettings collectorSettings) { + this.fileSettings = fileSettings; + this.collectorSettings = collectorSettings; + } + + @TikTokEventObserver + private void onResponse(LiveClient liveClient, TikTokWebsocketResponseEvent event) { + includeResponse(liveClient, event.getResponse()); + event.getResponse().getMessagesList().forEach(message -> includeMessage(liveClient, message)); + } + + @TikTokEventObserver + private void onEvent(LiveClient liveClient, TikTokEvent event) { + if (event instanceof TikTokConnectingEvent) { + sessionId = UUID.randomUUID().toString(); + userName = liveClient.getRoomInfo().getHostName(); + } + + if (event instanceof TikTokErrorEvent) { + return; + } + + includeEvent(event); + } + + @TikTokEventObserver + private void onError(LiveClient liveClient, TikTokErrorEvent event) { + event.getException().printStackTrace(); + includeError(event); + } + + + private void includeResponse(LiveClient liveClient, WebcastResponse message) { + var messageContent = Base64.getEncoder().encodeToString(message.toByteArray()); + saveJson(createJson("response", "webcast", messageContent)); + } + + private void includeMessage(LiveClient liveClient, WebcastResponse.Message message) { + var method = message.getMethod(); + var messageContent = Base64.getEncoder().encodeToString(message.getPayload().toByteArray()); + + saveJson(createJson("message", method, messageContent)); + } + + private void includeEvent(TikTokEvent event) { + var json = JsonUtil.toJson(event); + var content = Base64.getEncoder().encodeToString(json.getBytes()); + var name = event.getClass().getSimpleName(); + saveJson(createJson("event", name, content)); + } + + private void includeError(TikTokErrorEvent event) { + var exception = event.getException(); + var exceptionName = event.getException().getClass().getSimpleName(); + + var sw = new StringWriter(); + var pw = new PrintWriter(sw); + event.getException().printStackTrace(pw); + var content = sw.toString(); + + var json = createJson("error", exceptionName, content); + if (exception instanceof TikTokLiveMessageException ex) { + json.addProperty("message", ex.messageToBase64()); + json.addProperty("response", ex.webcastResponseToBase64()); + } + saveJson(json); + } + + private void saveJson(JsonObject jsonObject) { + if (!collectorSettings.getFilter().apply(jsonObject)) { + return; + } + try { + File file = new File(fileSettings.getParentFile(), jsonObject.get("dataType").getAsString()+":"+jsonObject.get("dataTypeName").getAsString()+".txt"); + file.createNewFile(); + Files.writeString(file.toPath(), jsonObject.toString(), StandardOpenOption.APPEND); + } catch (IOException e) { + e.printStackTrace(); + } + } + + private JsonObject createJson(String dataType, String dataTypeName, String content) { + JsonObject data = new JsonObject(); + data.addProperty("session", sessionId); + for (var entry : collectorSettings.getExtraFields().entrySet()) { + if (entry.getValue() instanceof JsonElement element) + data.add(entry.getKey(), element); + else + data.addProperty(entry.getKey(), entry.getValue().toString()); + } + data.addProperty("tiktokUser", userName); + data.addProperty("dataType", dataType); + data.addProperty("dataTypeName", dataTypeName); + data.addProperty("content", content); + return data; + } +} \ No newline at end of file diff --git a/extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/impl/TikTokLiveDataCollector.java b/extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/impl/MongoDataCollector.java similarity index 80% rename from extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/impl/TikTokLiveDataCollector.java rename to extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/impl/MongoDataCollector.java index d9899b7..095d0d1 100644 --- a/extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/impl/TikTokLiveDataCollector.java +++ b/extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/impl/MongoDataCollector.java @@ -32,25 +32,23 @@ import com.mongodb.client.MongoCollection; import com.mongodb.client.MongoDatabase; import com.mongodb.client.model.Indexes; import io.github.jwdeveloper.tiktok.extension.collector.api.data.CollectorListenerSettings; -import io.github.jwdeveloper.tiktok.extension.collector.api.data.LiveDataCollectorSettings; +import io.github.jwdeveloper.tiktok.extension.collector.api.mongo.MongoDataCollectorSettings; import org.bson.Document; import java.util.Map; -import java.util.TreeMap; import java.util.function.Function; -public class TikTokLiveDataCollector { +public class MongoDataCollector { - private final LiveDataCollectorSettings settings; + private final MongoDataCollectorSettings settings; private MongoClient mongoClient; private MongoDatabase database; private MongoCollection collection; - public TikTokLiveDataCollector(LiveDataCollectorSettings settings) { + public MongoDataCollector(MongoDataCollectorSettings settings) { this.settings = settings; } - public void connectDatabase() { var serverApi = ServerApi.builder() .version(ServerApiVersion.V1) @@ -72,19 +70,19 @@ public class TikTokLiveDataCollector { mongoClient.close(); } - public TikTokLiveDataCollectorListener newListener() { + public MongoDataCollectorListener newListener() { return newListener(Map.of()); } - public TikTokLiveDataCollectorListener newListener(Map additionalFields) { + public MongoDataCollectorListener newListener(Map additionalFields) { return newListener(additionalFields, (e)->true); } - public TikTokLiveDataCollectorListener newListener(Map additionalFields, - Function filter) { + public MongoDataCollectorListener newListener(Map additionalFields, + Function filter) { var settings = new CollectorListenerSettings(); settings.setExtraFields(additionalFields); settings.setFilter(filter); - return new TikTokLiveDataCollectorListener(collection, settings); + return new MongoDataCollectorListener(collection, settings); } -} +} \ No newline at end of file diff --git a/extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/impl/TikTokLiveDataCollectorListener.java b/extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/impl/MongoDataCollectorListener.java similarity index 94% rename from extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/impl/TikTokLiveDataCollectorListener.java rename to extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/impl/MongoDataCollectorListener.java index 17d9578..daa922c 100644 --- a/extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/impl/TikTokLiveDataCollectorListener.java +++ b/extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/impl/MongoDataCollectorListener.java @@ -19,14 +19,14 @@ import java.io.StringWriter; import java.util.Base64; import java.util.UUID; -public class TikTokLiveDataCollectorListener implements LiveDataCollector { +public class MongoDataCollectorListener implements LiveDataCollector { private final MongoCollection collection; private final CollectorListenerSettings settings; private String sessionId; private String userName; - public TikTokLiveDataCollectorListener(MongoCollection collection, CollectorListenerSettings settings) { + public MongoDataCollectorListener(MongoCollection collection, CollectorListenerSettings settings) { this.collection = collection; this.settings = settings; } @@ -35,10 +35,7 @@ public class TikTokLiveDataCollectorListener implements LiveDataCollector { @TikTokEventObserver private void onResponse(LiveClient liveClient, TikTokWebsocketResponseEvent event) { includeResponse(liveClient, event.getResponse()); - event.getResponse().getMessagesList().forEach(message -> - { - includeMessage(liveClient, message); - }); + event.getResponse().getMessagesList().forEach(message -> includeMessage(liveClient, message)); } @TikTokEventObserver @@ -119,4 +116,4 @@ public class TikTokLiveDataCollectorListener implements LiveDataCollector { doc.append("content", content); return doc; } -} +} \ No newline at end of file