From 3ae73072ffd768314bcc20750f9bafc2a499f1ba Mon Sep 17 00:00:00 2001 From: kohlerpop1 Date: Wed, 28 Feb 2024 12:24:34 -0500 Subject: [PATCH 1/2] Working on collecting to files! --- .../collector/TikTokLiveCollector.java | 22 ++-- .../api/data/CollectorListenerSettings.java | 5 +- .../api/file/FileDataCollectorSettings.java | 33 +++++ .../MongoDBConnectionStringBuilder.java | 29 +--- .../MongoDataCollectorSettings.java} | 13 +- .../collector/impl/FileDataCollector.java | 54 ++++++++ .../impl/FileDataCollectorListener.java | 124 ++++++++++++++++++ ...Collector.java => MongoDataCollector.java} | 22 ++-- ...r.java => MongoDataCollectorListener.java} | 11 +- 9 files changed, 252 insertions(+), 61 deletions(-) create mode 100644 extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/api/file/FileDataCollectorSettings.java rename extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/api/{data => mongo}/MongoDBConnectionStringBuilder.java (71%) rename extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/api/{data/LiveDataCollectorSettings.java => mongo/MongoDataCollectorSettings.java} (86%) create mode 100644 extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/impl/FileDataCollector.java create mode 100644 extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/impl/FileDataCollectorListener.java rename extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/impl/{TikTokLiveDataCollector.java => MongoDataCollector.java} (80%) rename extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/impl/{TikTokLiveDataCollectorListener.java => MongoDataCollectorListener.java} (94%) diff --git a/extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/TikTokLiveCollector.java b/extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/TikTokLiveCollector.java index 0ba1894..8b30f42 100644 --- a/extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/TikTokLiveCollector.java +++ b/extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/TikTokLiveCollector.java @@ -22,19 +22,25 @@ */ package io.github.jwdeveloper.tiktok.extension.collector; -import io.github.jwdeveloper.tiktok.extension.collector.api.LiveDataCollector; -import io.github.jwdeveloper.tiktok.extension.collector.api.data.LiveDataCollectorSettings; -import io.github.jwdeveloper.tiktok.extension.collector.impl.TikTokLiveDataCollector; +import io.github.jwdeveloper.tiktok.extension.collector.api.file.FileDataCollectorSettings; +import io.github.jwdeveloper.tiktok.extension.collector.api.mongo.MongoDataCollectorSettings; +import io.github.jwdeveloper.tiktok.extension.collector.impl.*; import java.util.function.Consumer; public class TikTokLiveCollector { - - public static TikTokLiveDataCollector use(Consumer consumer) + public static MongoDataCollector useMongo(Consumer consumer) { - var settings = new LiveDataCollectorSettings(); + var settings = new MongoDataCollectorSettings(); consumer.accept(settings); - return new TikTokLiveDataCollector(settings); + return new MongoDataCollector(settings); } -} + + public static FileDataCollector useFile(Consumer consumer) + { + var settings = new FileDataCollectorSettings(); + consumer.accept(settings); + return new FileDataCollector(settings); + } +} \ No newline at end of file diff --git a/extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/api/data/CollectorListenerSettings.java b/extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/api/data/CollectorListenerSettings.java index 8399a05..7c4710c 100644 --- a/extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/api/data/CollectorListenerSettings.java +++ b/extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/api/data/CollectorListenerSettings.java @@ -1,7 +1,6 @@ package io.github.jwdeveloper.tiktok.extension.collector.api.data; import lombok.Data; -import org.bson.Document; import java.util.Map; import java.util.function.Function; @@ -9,5 +8,5 @@ import java.util.function.Function; @Data public class CollectorListenerSettings { private Map extraFields; - private Function filter; -} + private Function filter; +} \ No newline at end of file diff --git a/extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/api/file/FileDataCollectorSettings.java b/extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/api/file/FileDataCollectorSettings.java new file mode 100644 index 0000000..395b8fe --- /dev/null +++ b/extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/api/file/FileDataCollectorSettings.java @@ -0,0 +1,33 @@ +/* + * Copyright (c) 2023-2023 jwdeveloper jacekwoln@gmail.com + * + * Permission is hereby granted, free of charge, to any person obtaining + * a copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE + * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION + * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package io.github.jwdeveloper.tiktok.extension.collector.api.file; + +import lombok.Data; + +import java.io.File; + +@Data +public class FileDataCollectorSettings { + + private File parentFile; +} \ No newline at end of file diff --git a/extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/api/data/MongoDBConnectionStringBuilder.java b/extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/api/mongo/MongoDBConnectionStringBuilder.java similarity index 71% rename from extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/api/data/MongoDBConnectionStringBuilder.java rename to extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/api/mongo/MongoDBConnectionStringBuilder.java index 556a0cd..e1b4b85 100644 --- a/extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/api/data/MongoDBConnectionStringBuilder.java +++ b/extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/api/mongo/MongoDBConnectionStringBuilder.java @@ -20,36 +20,21 @@ * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package io.github.jwdeveloper.tiktok.extension.collector.api.data; +package io.github.jwdeveloper.tiktok.extension.collector.api.mongo; +import lombok.Setter; +import lombok.experimental.Accessors; + +@Setter +@Accessors(chain = true) public class MongoDBConnectionStringBuilder { private String username; private String password; private String database; private String cluster; - public MongoDBConnectionStringBuilder setUsername(String username) { - this.username = username; - return this; - } - - public MongoDBConnectionStringBuilder setPassword(String password) { - this.password = password; - return this; - } - - public MongoDBConnectionStringBuilder setDatabase(String database) { - this.database = database; - return this; - } - - public MongoDBConnectionStringBuilder setCluster(String cluster) { - this.cluster = cluster; - return this; - } - public String build() { return String.format("mongodb+srv://%s:%s@%s/%s?retryWrites=true&w=majority", username, password, cluster, database); } -} +} \ No newline at end of file diff --git a/extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/api/data/LiveDataCollectorSettings.java b/extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/api/mongo/MongoDataCollectorSettings.java similarity index 86% rename from extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/api/data/LiveDataCollectorSettings.java rename to extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/api/mongo/MongoDataCollectorSettings.java index 4597c5c..bdabb40 100644 --- a/extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/api/data/LiveDataCollectorSettings.java +++ b/extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/api/mongo/MongoDataCollectorSettings.java @@ -20,14 +20,14 @@ * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package io.github.jwdeveloper.tiktok.extension.collector.api.data; +package io.github.jwdeveloper.tiktok.extension.collector.api.mongo; -import lombok.Data; +import lombok.*; import java.util.function.Consumer; @Data -public class LiveDataCollectorSettings { +public class MongoDataCollectorSettings { private String connectionUrl; @@ -35,14 +35,9 @@ public class LiveDataCollectorSettings { private String sessionTag; - - public void setConnectionUrl(String connectionUrl) { - this.connectionUrl = connectionUrl; - } - public void setConnectionUrl(Consumer consumer) { var builder = new MongoDBConnectionStringBuilder(); consumer.accept(builder); connectionUrl = builder.build(); } -} +} \ No newline at end of file diff --git a/extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/impl/FileDataCollector.java b/extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/impl/FileDataCollector.java new file mode 100644 index 0000000..4ea9c1d --- /dev/null +++ b/extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/impl/FileDataCollector.java @@ -0,0 +1,54 @@ +/* + * Copyright (c) 2023-2023 jwdeveloper jacekwoln@gmail.com + * + * Permission is hereby granted, free of charge, to any person obtaining + * a copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE + * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION + * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package io.github.jwdeveloper.tiktok.extension.collector.impl; + +import io.github.jwdeveloper.tiktok.extension.collector.api.data.CollectorListenerSettings; +import io.github.jwdeveloper.tiktok.extension.collector.api.file.FileDataCollectorSettings; + +import java.util.Map; +import java.util.function.Function; + +public class FileDataCollector { + + private final FileDataCollectorSettings settings; + + public FileDataCollector(FileDataCollectorSettings settings) { + this.settings = settings; + } + + public FileDataCollectorListener newListener() { + return newListener(Map.of()); + } + + public FileDataCollectorListener newListener(Map additionalFields) { + return newListener(additionalFields, (e)->true); + } + + public FileDataCollectorListener newListener(Map additionalFields, + Function filter) { + var settings = new CollectorListenerSettings(); + settings.setExtraFields(additionalFields); + settings.setFilter(filter); + return new FileDataCollectorListener(this.settings, settings); + } +} \ No newline at end of file diff --git a/extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/impl/FileDataCollectorListener.java b/extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/impl/FileDataCollectorListener.java new file mode 100644 index 0000000..e724285 --- /dev/null +++ b/extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/impl/FileDataCollectorListener.java @@ -0,0 +1,124 @@ +package io.github.jwdeveloper.tiktok.extension.collector.impl; + +import com.google.gson.*; +import io.github.jwdeveloper.tiktok.annotations.TikTokEventObserver; +import io.github.jwdeveloper.tiktok.data.events.TikTokErrorEvent; +import io.github.jwdeveloper.tiktok.data.events.common.TikTokEvent; +import io.github.jwdeveloper.tiktok.data.events.control.TikTokConnectingEvent; +import io.github.jwdeveloper.tiktok.data.events.websocket.TikTokWebsocketResponseEvent; +import io.github.jwdeveloper.tiktok.exceptions.TikTokLiveMessageException; +import io.github.jwdeveloper.tiktok.extension.collector.api.LiveDataCollector; +import io.github.jwdeveloper.tiktok.extension.collector.api.data.CollectorListenerSettings; +import io.github.jwdeveloper.tiktok.extension.collector.api.file.FileDataCollectorSettings; +import io.github.jwdeveloper.tiktok.live.LiveClient; +import io.github.jwdeveloper.tiktok.messages.webcast.WebcastResponse; +import io.github.jwdeveloper.tiktok.utils.JsonUtil; + +import java.io.*; +import java.nio.file.*; +import java.util.*; + +public class FileDataCollectorListener implements LiveDataCollector { + + private final FileDataCollectorSettings fileSettings; + private final CollectorListenerSettings collectorSettings; + private String sessionId; + private String userName; + + public FileDataCollectorListener(FileDataCollectorSettings fileSettings, CollectorListenerSettings collectorSettings) { + this.fileSettings = fileSettings; + this.collectorSettings = collectorSettings; + } + + @TikTokEventObserver + private void onResponse(LiveClient liveClient, TikTokWebsocketResponseEvent event) { + includeResponse(liveClient, event.getResponse()); + event.getResponse().getMessagesList().forEach(message -> includeMessage(liveClient, message)); + } + + @TikTokEventObserver + private void onEvent(LiveClient liveClient, TikTokEvent event) { + if (event instanceof TikTokConnectingEvent) { + sessionId = UUID.randomUUID().toString(); + userName = liveClient.getRoomInfo().getHostName(); + } + + if (event instanceof TikTokErrorEvent) { + return; + } + + includeEvent(event); + } + + @TikTokEventObserver + private void onError(LiveClient liveClient, TikTokErrorEvent event) { + event.getException().printStackTrace(); + includeError(event); + } + + + private void includeResponse(LiveClient liveClient, WebcastResponse message) { + var messageContent = Base64.getEncoder().encodeToString(message.toByteArray()); + saveJson(createJson("response", "webcast", messageContent)); + } + + private void includeMessage(LiveClient liveClient, WebcastResponse.Message message) { + var method = message.getMethod(); + var messageContent = Base64.getEncoder().encodeToString(message.getPayload().toByteArray()); + + saveJson(createJson("message", method, messageContent)); + } + + private void includeEvent(TikTokEvent event) { + var json = JsonUtil.toJson(event); + var content = Base64.getEncoder().encodeToString(json.getBytes()); + var name = event.getClass().getSimpleName(); + saveJson(createJson("event", name, content)); + } + + private void includeError(TikTokErrorEvent event) { + var exception = event.getException(); + var exceptionName = event.getException().getClass().getSimpleName(); + + var sw = new StringWriter(); + var pw = new PrintWriter(sw); + event.getException().printStackTrace(pw); + var content = sw.toString(); + + var json = createJson("error", exceptionName, content); + if (exception instanceof TikTokLiveMessageException ex) { + json.addProperty("message", ex.messageToBase64()); + json.addProperty("response", ex.webcastResponseToBase64()); + } + saveJson(json); + } + + private void saveJson(JsonObject jsonObject) { + if (!collectorSettings.getFilter().apply(jsonObject)) { + return; + } + try { + File file = new File(fileSettings.getParentFile(), jsonObject.get("dataType").getAsString()+":"+jsonObject.get("dataTypeName").getAsString()+".txt"); + file.createNewFile(); + Files.writeString(file.toPath(), jsonObject.toString(), StandardOpenOption.APPEND); + } catch (IOException e) { + e.printStackTrace(); + } + } + + private JsonObject createJson(String dataType, String dataTypeName, String content) { + JsonObject data = new JsonObject(); + data.addProperty("session", sessionId); + for (var entry : collectorSettings.getExtraFields().entrySet()) { + if (entry.getValue() instanceof JsonElement element) + data.add(entry.getKey(), element); + else + data.addProperty(entry.getKey(), entry.getValue().toString()); + } + data.addProperty("tiktokUser", userName); + data.addProperty("dataType", dataType); + data.addProperty("dataTypeName", dataTypeName); + data.addProperty("content", content); + return data; + } +} \ No newline at end of file diff --git a/extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/impl/TikTokLiveDataCollector.java b/extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/impl/MongoDataCollector.java similarity index 80% rename from extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/impl/TikTokLiveDataCollector.java rename to extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/impl/MongoDataCollector.java index d9899b7..095d0d1 100644 --- a/extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/impl/TikTokLiveDataCollector.java +++ b/extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/impl/MongoDataCollector.java @@ -32,25 +32,23 @@ import com.mongodb.client.MongoCollection; import com.mongodb.client.MongoDatabase; import com.mongodb.client.model.Indexes; import io.github.jwdeveloper.tiktok.extension.collector.api.data.CollectorListenerSettings; -import io.github.jwdeveloper.tiktok.extension.collector.api.data.LiveDataCollectorSettings; +import io.github.jwdeveloper.tiktok.extension.collector.api.mongo.MongoDataCollectorSettings; import org.bson.Document; import java.util.Map; -import java.util.TreeMap; import java.util.function.Function; -public class TikTokLiveDataCollector { +public class MongoDataCollector { - private final LiveDataCollectorSettings settings; + private final MongoDataCollectorSettings settings; private MongoClient mongoClient; private MongoDatabase database; private MongoCollection collection; - public TikTokLiveDataCollector(LiveDataCollectorSettings settings) { + public MongoDataCollector(MongoDataCollectorSettings settings) { this.settings = settings; } - public void connectDatabase() { var serverApi = ServerApi.builder() .version(ServerApiVersion.V1) @@ -72,19 +70,19 @@ public class TikTokLiveDataCollector { mongoClient.close(); } - public TikTokLiveDataCollectorListener newListener() { + public MongoDataCollectorListener newListener() { return newListener(Map.of()); } - public TikTokLiveDataCollectorListener newListener(Map additionalFields) { + public MongoDataCollectorListener newListener(Map additionalFields) { return newListener(additionalFields, (e)->true); } - public TikTokLiveDataCollectorListener newListener(Map additionalFields, - Function filter) { + public MongoDataCollectorListener newListener(Map additionalFields, + Function filter) { var settings = new CollectorListenerSettings(); settings.setExtraFields(additionalFields); settings.setFilter(filter); - return new TikTokLiveDataCollectorListener(collection, settings); + return new MongoDataCollectorListener(collection, settings); } -} +} \ No newline at end of file diff --git a/extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/impl/TikTokLiveDataCollectorListener.java b/extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/impl/MongoDataCollectorListener.java similarity index 94% rename from extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/impl/TikTokLiveDataCollectorListener.java rename to extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/impl/MongoDataCollectorListener.java index 17d9578..daa922c 100644 --- a/extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/impl/TikTokLiveDataCollectorListener.java +++ b/extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/impl/MongoDataCollectorListener.java @@ -19,14 +19,14 @@ import java.io.StringWriter; import java.util.Base64; import java.util.UUID; -public class TikTokLiveDataCollectorListener implements LiveDataCollector { +public class MongoDataCollectorListener implements LiveDataCollector { private final MongoCollection collection; private final CollectorListenerSettings settings; private String sessionId; private String userName; - public TikTokLiveDataCollectorListener(MongoCollection collection, CollectorListenerSettings settings) { + public MongoDataCollectorListener(MongoCollection collection, CollectorListenerSettings settings) { this.collection = collection; this.settings = settings; } @@ -35,10 +35,7 @@ public class TikTokLiveDataCollectorListener implements LiveDataCollector { @TikTokEventObserver private void onResponse(LiveClient liveClient, TikTokWebsocketResponseEvent event) { includeResponse(liveClient, event.getResponse()); - event.getResponse().getMessagesList().forEach(message -> - { - includeMessage(liveClient, message); - }); + event.getResponse().getMessagesList().forEach(message -> includeMessage(liveClient, message)); } @TikTokEventObserver @@ -119,4 +116,4 @@ public class TikTokLiveDataCollectorListener implements LiveDataCollector { doc.append("content", content); return doc; } -} +} \ No newline at end of file From b82c7184b3b21537b2b3af5ded4220d87761594b Mon Sep 17 00:00:00 2001 From: JW Date: Fri, 1 Mar 2024 01:52:54 +0100 Subject: [PATCH 2/2] Removed unused projects. --- .../jwdeveloper/tiktok/CollectorExample.java | 7 +- .../collector/TikTokLiveCollector.java | 25 ++-- .../extension/collector/api/Storage.java | 11 ++ .../CollectorListenerSettings.java | 5 +- .../FileDataCollectorSettings.java | 2 +- .../mongo/MongoDBConnectionStringBuilder.java | 2 +- .../mongo/MongoDataCollectorSettings.java | 9 +- ...eDataCollector.java => DataCollector.java} | 32 +++-- ...stener.java => DataCollectorListener.java} | 14 +- .../impl/FileDataCollectorListener.java | 124 ------------------ .../collector/impl/MongoDataCollector.java | 88 ------------- .../collector/impl/storages/FileStorage.java | 41 ++++++ .../collector/impl/storages/MongoStorage.java | 58 ++++++++ 13 files changed, 167 insertions(+), 251 deletions(-) create mode 100644 extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/api/Storage.java rename extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/api/{data => settings}/CollectorListenerSettings.java (54%) rename extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/api/{file => settings}/FileDataCollectorSettings.java (94%) rename extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/api/{ => settings}/mongo/MongoDBConnectionStringBuilder.java (95%) rename extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/api/{ => settings}/mongo/MongoDataCollectorSettings.java (84%) rename extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/impl/{FileDataCollector.java => DataCollector.java} (62%) rename extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/impl/{MongoDataCollectorListener.java => DataCollectorListener.java} (89%) delete mode 100644 extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/impl/FileDataCollectorListener.java delete mode 100644 extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/impl/MongoDataCollector.java create mode 100644 extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/impl/storages/FileStorage.java create mode 100644 extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/impl/storages/MongoStorage.java diff --git a/Examples/src/main/java/io/github/jwdeveloper/tiktok/CollectorExample.java b/Examples/src/main/java/io/github/jwdeveloper/tiktok/CollectorExample.java index 98e52c4..4c842cc 100644 --- a/Examples/src/main/java/io/github/jwdeveloper/tiktok/CollectorExample.java +++ b/Examples/src/main/java/io/github/jwdeveloper/tiktok/CollectorExample.java @@ -40,12 +40,11 @@ public class CollectorExample { public static void main(String[] args) throws IOException { - var collector = TikTokLiveCollector.use(settings -> + var collector = TikTokLiveCollector.useMongo(settings -> { settings.setConnectionUrl("mongodb+srv://" + mongoUser + ":" + mongoPassword + "@" + mongoDatabase + "/?retryWrites=true&w=majority"); - settings.setDatabaseName("tiktok"); }); - collector.connectDatabase(); + collector.connect(); var users = List.of("tehila_723", "dino123597", "domaxyzx", "dash4214", "obserwacje_live"); Map additionalDataFields = Map.of("sessionTag", "ExampleTag"); @@ -71,6 +70,6 @@ public class CollectorExample { } System.in.read(); - collector.disconnectDatabase(); + collector.disconnect(); } } diff --git a/extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/TikTokLiveCollector.java b/extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/TikTokLiveCollector.java index 8b30f42..3128dc9 100644 --- a/extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/TikTokLiveCollector.java +++ b/extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/TikTokLiveCollector.java @@ -22,25 +22,34 @@ */ package io.github.jwdeveloper.tiktok.extension.collector; -import io.github.jwdeveloper.tiktok.extension.collector.api.file.FileDataCollectorSettings; -import io.github.jwdeveloper.tiktok.extension.collector.api.mongo.MongoDataCollectorSettings; +import io.github.jwdeveloper.tiktok.extension.collector.api.settings.FileDataCollectorSettings; +import io.github.jwdeveloper.tiktok.extension.collector.api.settings.mongo.MongoDataCollectorSettings; import io.github.jwdeveloper.tiktok.extension.collector.impl.*; +import io.github.jwdeveloper.tiktok.extension.collector.impl.storages.FileStorage; +import io.github.jwdeveloper.tiktok.extension.collector.impl.storages.MongoStorage; import java.util.function.Consumer; + +/** + * + */ public class TikTokLiveCollector { - public static MongoDataCollector useMongo(Consumer consumer) - { + + public static DataCollector useMongo(Consumer consumer) { var settings = new MongoDataCollectorSettings(); consumer.accept(settings); - return new MongoDataCollector(settings); + + var storage = new MongoStorage(settings); + return new DataCollector(storage); } - public static FileDataCollector useFile(Consumer consumer) - { + public static DataCollector useFile(Consumer consumer) { var settings = new FileDataCollectorSettings(); consumer.accept(settings); - return new FileDataCollector(settings); + + var storage = new FileStorage(settings); + return new DataCollector(storage); } } \ No newline at end of file diff --git a/extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/api/Storage.java b/extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/api/Storage.java new file mode 100644 index 0000000..dc7e1a0 --- /dev/null +++ b/extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/api/Storage.java @@ -0,0 +1,11 @@ +package io.github.jwdeveloper.tiktok.extension.collector.api; + +import org.bson.Document; + +public interface Storage { + void connect(); + + void disconnect(); + + void insert(Document document); +} diff --git a/extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/api/data/CollectorListenerSettings.java b/extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/api/settings/CollectorListenerSettings.java similarity index 54% rename from extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/api/data/CollectorListenerSettings.java rename to extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/api/settings/CollectorListenerSettings.java index 7c4710c..09d09d8 100644 --- a/extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/api/data/CollectorListenerSettings.java +++ b/extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/api/settings/CollectorListenerSettings.java @@ -1,6 +1,7 @@ -package io.github.jwdeveloper.tiktok.extension.collector.api.data; +package io.github.jwdeveloper.tiktok.extension.collector.api.settings; import lombok.Data; +import org.bson.Document; import java.util.Map; import java.util.function.Function; @@ -8,5 +9,5 @@ import java.util.function.Function; @Data public class CollectorListenerSettings { private Map extraFields; - private Function filter; + private Function filter; } \ No newline at end of file diff --git a/extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/api/file/FileDataCollectorSettings.java b/extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/api/settings/FileDataCollectorSettings.java similarity index 94% rename from extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/api/file/FileDataCollectorSettings.java rename to extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/api/settings/FileDataCollectorSettings.java index 395b8fe..0eee68f 100644 --- a/extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/api/file/FileDataCollectorSettings.java +++ b/extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/api/settings/FileDataCollectorSettings.java @@ -20,7 +20,7 @@ * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package io.github.jwdeveloper.tiktok.extension.collector.api.file; +package io.github.jwdeveloper.tiktok.extension.collector.api.settings; import lombok.Data; diff --git a/extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/api/mongo/MongoDBConnectionStringBuilder.java b/extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/api/settings/mongo/MongoDBConnectionStringBuilder.java similarity index 95% rename from extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/api/mongo/MongoDBConnectionStringBuilder.java rename to extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/api/settings/mongo/MongoDBConnectionStringBuilder.java index e1b4b85..9ae8a3c 100644 --- a/extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/api/mongo/MongoDBConnectionStringBuilder.java +++ b/extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/api/settings/mongo/MongoDBConnectionStringBuilder.java @@ -20,7 +20,7 @@ * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package io.github.jwdeveloper.tiktok.extension.collector.api.mongo; +package io.github.jwdeveloper.tiktok.extension.collector.api.settings.mongo; import lombok.Setter; import lombok.experimental.Accessors; diff --git a/extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/api/mongo/MongoDataCollectorSettings.java b/extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/api/settings/mongo/MongoDataCollectorSettings.java similarity index 84% rename from extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/api/mongo/MongoDataCollectorSettings.java rename to extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/api/settings/mongo/MongoDataCollectorSettings.java index bdabb40..3d319ab 100644 --- a/extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/api/mongo/MongoDataCollectorSettings.java +++ b/extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/api/settings/mongo/MongoDataCollectorSettings.java @@ -20,7 +20,7 @@ * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package io.github.jwdeveloper.tiktok.extension.collector.api.mongo; +package io.github.jwdeveloper.tiktok.extension.collector.api.settings.mongo; import lombok.*; @@ -29,13 +29,14 @@ import java.util.function.Consumer; @Data public class MongoDataCollectorSettings { + @Setter private String connectionUrl; - private String databaseName; + private String databaseName = "tiktok"; - private String sessionTag; + private String collectionName = "data"; - public void setConnectionUrl(Consumer consumer) { + public void connectionBuilder(Consumer consumer) { var builder = new MongoDBConnectionStringBuilder(); consumer.accept(builder); connectionUrl = builder.build(); diff --git a/extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/impl/FileDataCollector.java b/extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/impl/DataCollector.java similarity index 62% rename from extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/impl/FileDataCollector.java rename to extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/impl/DataCollector.java index 4ea9c1d..27b3f08 100644 --- a/extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/impl/FileDataCollector.java +++ b/extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/impl/DataCollector.java @@ -22,33 +22,41 @@ */ package io.github.jwdeveloper.tiktok.extension.collector.impl; -import io.github.jwdeveloper.tiktok.extension.collector.api.data.CollectorListenerSettings; -import io.github.jwdeveloper.tiktok.extension.collector.api.file.FileDataCollectorSettings; +import io.github.jwdeveloper.tiktok.extension.collector.api.Storage; +import io.github.jwdeveloper.tiktok.extension.collector.api.settings.CollectorListenerSettings; +import org.bson.Document; import java.util.Map; import java.util.function.Function; -public class FileDataCollector { +public class DataCollector { - private final FileDataCollectorSettings settings; + private final Storage storage; - public FileDataCollector(FileDataCollectorSettings settings) { - this.settings = settings; + public DataCollector(Storage storage) { + this.storage = storage; } - public FileDataCollectorListener newListener() { + public void connect() { + storage.connect(); + } + public void disconnect() { + storage.disconnect(); + } + + public DataCollectorListener newListener() { return newListener(Map.of()); } - public FileDataCollectorListener newListener(Map additionalFields) { - return newListener(additionalFields, (e)->true); + public DataCollectorListener newListener(Map additionalFields) { + return newListener(additionalFields, (e) -> true); } - public FileDataCollectorListener newListener(Map additionalFields, - Function filter) { + public DataCollectorListener newListener(Map additionalFields, + Function filter) { var settings = new CollectorListenerSettings(); settings.setExtraFields(additionalFields); settings.setFilter(filter); - return new FileDataCollectorListener(this.settings, settings); + return new DataCollectorListener(storage, settings); } } \ No newline at end of file diff --git a/extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/impl/MongoDataCollectorListener.java b/extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/impl/DataCollectorListener.java similarity index 89% rename from extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/impl/MongoDataCollectorListener.java rename to extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/impl/DataCollectorListener.java index daa922c..1f57157 100644 --- a/extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/impl/MongoDataCollectorListener.java +++ b/extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/impl/DataCollectorListener.java @@ -1,6 +1,5 @@ package io.github.jwdeveloper.tiktok.extension.collector.impl; -import com.mongodb.client.MongoCollection; import io.github.jwdeveloper.tiktok.annotations.TikTokEventObserver; import io.github.jwdeveloper.tiktok.data.events.TikTokErrorEvent; import io.github.jwdeveloper.tiktok.data.events.common.TikTokEvent; @@ -8,7 +7,8 @@ import io.github.jwdeveloper.tiktok.data.events.control.TikTokConnectingEvent; import io.github.jwdeveloper.tiktok.data.events.websocket.TikTokWebsocketResponseEvent; import io.github.jwdeveloper.tiktok.exceptions.TikTokLiveMessageException; import io.github.jwdeveloper.tiktok.extension.collector.api.LiveDataCollector; -import io.github.jwdeveloper.tiktok.extension.collector.api.data.CollectorListenerSettings; +import io.github.jwdeveloper.tiktok.extension.collector.api.Storage; +import io.github.jwdeveloper.tiktok.extension.collector.api.settings.CollectorListenerSettings; import io.github.jwdeveloper.tiktok.live.LiveClient; import io.github.jwdeveloper.tiktok.messages.webcast.WebcastResponse; import io.github.jwdeveloper.tiktok.utils.JsonUtil; @@ -19,15 +19,15 @@ import java.io.StringWriter; import java.util.Base64; import java.util.UUID; -public class MongoDataCollectorListener implements LiveDataCollector { +public class DataCollectorListener implements LiveDataCollector { - private final MongoCollection collection; + private final Storage storage; private final CollectorListenerSettings settings; private String sessionId; private String userName; - public MongoDataCollectorListener(MongoCollection collection, CollectorListenerSettings settings) { - this.collection = collection; + public DataCollectorListener(Storage collection, CollectorListenerSettings settings) { + this.storage = collection; this.settings = settings; } @@ -100,7 +100,7 @@ public class MongoDataCollectorListener implements LiveDataCollector { if (!settings.getFilter().apply(document)) { return; } - collection.insertOne(document); + storage.insert(document); } diff --git a/extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/impl/FileDataCollectorListener.java b/extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/impl/FileDataCollectorListener.java deleted file mode 100644 index e724285..0000000 --- a/extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/impl/FileDataCollectorListener.java +++ /dev/null @@ -1,124 +0,0 @@ -package io.github.jwdeveloper.tiktok.extension.collector.impl; - -import com.google.gson.*; -import io.github.jwdeveloper.tiktok.annotations.TikTokEventObserver; -import io.github.jwdeveloper.tiktok.data.events.TikTokErrorEvent; -import io.github.jwdeveloper.tiktok.data.events.common.TikTokEvent; -import io.github.jwdeveloper.tiktok.data.events.control.TikTokConnectingEvent; -import io.github.jwdeveloper.tiktok.data.events.websocket.TikTokWebsocketResponseEvent; -import io.github.jwdeveloper.tiktok.exceptions.TikTokLiveMessageException; -import io.github.jwdeveloper.tiktok.extension.collector.api.LiveDataCollector; -import io.github.jwdeveloper.tiktok.extension.collector.api.data.CollectorListenerSettings; -import io.github.jwdeveloper.tiktok.extension.collector.api.file.FileDataCollectorSettings; -import io.github.jwdeveloper.tiktok.live.LiveClient; -import io.github.jwdeveloper.tiktok.messages.webcast.WebcastResponse; -import io.github.jwdeveloper.tiktok.utils.JsonUtil; - -import java.io.*; -import java.nio.file.*; -import java.util.*; - -public class FileDataCollectorListener implements LiveDataCollector { - - private final FileDataCollectorSettings fileSettings; - private final CollectorListenerSettings collectorSettings; - private String sessionId; - private String userName; - - public FileDataCollectorListener(FileDataCollectorSettings fileSettings, CollectorListenerSettings collectorSettings) { - this.fileSettings = fileSettings; - this.collectorSettings = collectorSettings; - } - - @TikTokEventObserver - private void onResponse(LiveClient liveClient, TikTokWebsocketResponseEvent event) { - includeResponse(liveClient, event.getResponse()); - event.getResponse().getMessagesList().forEach(message -> includeMessage(liveClient, message)); - } - - @TikTokEventObserver - private void onEvent(LiveClient liveClient, TikTokEvent event) { - if (event instanceof TikTokConnectingEvent) { - sessionId = UUID.randomUUID().toString(); - userName = liveClient.getRoomInfo().getHostName(); - } - - if (event instanceof TikTokErrorEvent) { - return; - } - - includeEvent(event); - } - - @TikTokEventObserver - private void onError(LiveClient liveClient, TikTokErrorEvent event) { - event.getException().printStackTrace(); - includeError(event); - } - - - private void includeResponse(LiveClient liveClient, WebcastResponse message) { - var messageContent = Base64.getEncoder().encodeToString(message.toByteArray()); - saveJson(createJson("response", "webcast", messageContent)); - } - - private void includeMessage(LiveClient liveClient, WebcastResponse.Message message) { - var method = message.getMethod(); - var messageContent = Base64.getEncoder().encodeToString(message.getPayload().toByteArray()); - - saveJson(createJson("message", method, messageContent)); - } - - private void includeEvent(TikTokEvent event) { - var json = JsonUtil.toJson(event); - var content = Base64.getEncoder().encodeToString(json.getBytes()); - var name = event.getClass().getSimpleName(); - saveJson(createJson("event", name, content)); - } - - private void includeError(TikTokErrorEvent event) { - var exception = event.getException(); - var exceptionName = event.getException().getClass().getSimpleName(); - - var sw = new StringWriter(); - var pw = new PrintWriter(sw); - event.getException().printStackTrace(pw); - var content = sw.toString(); - - var json = createJson("error", exceptionName, content); - if (exception instanceof TikTokLiveMessageException ex) { - json.addProperty("message", ex.messageToBase64()); - json.addProperty("response", ex.webcastResponseToBase64()); - } - saveJson(json); - } - - private void saveJson(JsonObject jsonObject) { - if (!collectorSettings.getFilter().apply(jsonObject)) { - return; - } - try { - File file = new File(fileSettings.getParentFile(), jsonObject.get("dataType").getAsString()+":"+jsonObject.get("dataTypeName").getAsString()+".txt"); - file.createNewFile(); - Files.writeString(file.toPath(), jsonObject.toString(), StandardOpenOption.APPEND); - } catch (IOException e) { - e.printStackTrace(); - } - } - - private JsonObject createJson(String dataType, String dataTypeName, String content) { - JsonObject data = new JsonObject(); - data.addProperty("session", sessionId); - for (var entry : collectorSettings.getExtraFields().entrySet()) { - if (entry.getValue() instanceof JsonElement element) - data.add(entry.getKey(), element); - else - data.addProperty(entry.getKey(), entry.getValue().toString()); - } - data.addProperty("tiktokUser", userName); - data.addProperty("dataType", dataType); - data.addProperty("dataTypeName", dataTypeName); - data.addProperty("content", content); - return data; - } -} \ No newline at end of file diff --git a/extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/impl/MongoDataCollector.java b/extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/impl/MongoDataCollector.java deleted file mode 100644 index 095d0d1..0000000 --- a/extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/impl/MongoDataCollector.java +++ /dev/null @@ -1,88 +0,0 @@ -/* - * Copyright (c) 2023-2023 jwdeveloper jacekwoln@gmail.com - * - * Permission is hereby granted, free of charge, to any person obtaining - * a copy of this software and associated documentation files (the - * "Software"), to deal in the Software without restriction, including - * without limitation the rights to use, copy, modify, merge, publish, - * distribute, sublicense, and/or sell copies of the Software, and to - * permit persons to whom the Software is furnished to do so, subject to - * the following conditions: - * - * The above copyright notice and this permission notice shall be - * included in all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE - * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION - * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION - * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - */ -package io.github.jwdeveloper.tiktok.extension.collector.impl; - -import com.mongodb.ConnectionString; -import com.mongodb.MongoClientSettings; -import com.mongodb.ServerApi; -import com.mongodb.ServerApiVersion; -import com.mongodb.client.MongoClient; -import com.mongodb.client.MongoClients; -import com.mongodb.client.MongoCollection; -import com.mongodb.client.MongoDatabase; -import com.mongodb.client.model.Indexes; -import io.github.jwdeveloper.tiktok.extension.collector.api.data.CollectorListenerSettings; -import io.github.jwdeveloper.tiktok.extension.collector.api.mongo.MongoDataCollectorSettings; -import org.bson.Document; - -import java.util.Map; -import java.util.function.Function; - -public class MongoDataCollector { - - private final MongoDataCollectorSettings settings; - private MongoClient mongoClient; - private MongoDatabase database; - private MongoCollection collection; - - public MongoDataCollector(MongoDataCollectorSettings settings) { - this.settings = settings; - } - - public void connectDatabase() { - var serverApi = ServerApi.builder() - .version(ServerApiVersion.V1) - .build(); - var mongoSettings = MongoClientSettings.builder() - .applyConnectionString(new ConnectionString(settings.getConnectionUrl())) - .serverApi(serverApi) - .build(); - - mongoClient = MongoClients.create(mongoSettings); - database = mongoClient.getDatabase(settings.getDatabaseName()); - collection = database.getCollection("data"); - collection.createIndex(Indexes.hashed("session")); - collection.createIndex(Indexes.hashed("dataType")); - } - - - public void disconnectDatabase() { - mongoClient.close(); - } - - public MongoDataCollectorListener newListener() { - return newListener(Map.of()); - } - - public MongoDataCollectorListener newListener(Map additionalFields) { - return newListener(additionalFields, (e)->true); - } - - public MongoDataCollectorListener newListener(Map additionalFields, - Function filter) { - var settings = new CollectorListenerSettings(); - settings.setExtraFields(additionalFields); - settings.setFilter(filter); - return new MongoDataCollectorListener(collection, settings); - } -} \ No newline at end of file diff --git a/extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/impl/storages/FileStorage.java b/extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/impl/storages/FileStorage.java new file mode 100644 index 0000000..64b4de2 --- /dev/null +++ b/extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/impl/storages/FileStorage.java @@ -0,0 +1,41 @@ +package io.github.jwdeveloper.tiktok.extension.collector.impl.storages; + +import io.github.jwdeveloper.tiktok.extension.collector.api.Storage; +import io.github.jwdeveloper.tiktok.extension.collector.api.settings.FileDataCollectorSettings; +import org.bson.Document; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.StandardOpenOption; + +public class FileStorage implements Storage { + + private final FileDataCollectorSettings settings; + + public FileStorage(FileDataCollectorSettings fileDataCollectorSettings) { + this.settings = fileDataCollectorSettings; + } + + @Override + public void connect() { + + } + + @Override + public void disconnect() { + + } + + @Override + public void insert(Document document) { + var fileName = document.get("dataType") + ":" + document.get("dataTypeName") + ".json"; + try { + var file = new File(settings.getParentFile(), fileName); + file.createNewFile(); + Files.writeString(file.toPath(), document.toJson(), StandardOpenOption.APPEND); + } catch (IOException e) { + e.printStackTrace(); + } + } +} diff --git a/extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/impl/storages/MongoStorage.java b/extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/impl/storages/MongoStorage.java new file mode 100644 index 0000000..5fef256 --- /dev/null +++ b/extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/impl/storages/MongoStorage.java @@ -0,0 +1,58 @@ +package io.github.jwdeveloper.tiktok.extension.collector.impl.storages; + +import com.mongodb.ConnectionString; +import com.mongodb.MongoClientSettings; +import com.mongodb.ServerApi; +import com.mongodb.ServerApiVersion; +import com.mongodb.client.MongoClient; +import com.mongodb.client.MongoClients; +import com.mongodb.client.MongoCollection; +import com.mongodb.client.MongoDatabase; +import com.mongodb.client.model.Indexes; +import io.github.jwdeveloper.tiktok.extension.collector.api.Storage; +import io.github.jwdeveloper.tiktok.extension.collector.api.settings.mongo.MongoDataCollectorSettings; +import org.bson.Document; + +public class MongoStorage implements Storage { + private MongoClient mongoClient; + private MongoDatabase database; + private MongoCollection collection; + private final MongoDataCollectorSettings settings; + + public MongoStorage(MongoDataCollectorSettings settings) { + this.settings = settings; + } + + @Override + public void connect() { + + var serverApi = ServerApi.builder() + .version(ServerApiVersion.V1) + .build(); + var mongoSettings = MongoClientSettings.builder() + .applyConnectionString(new ConnectionString(settings.getConnectionUrl())) + .serverApi(serverApi) + .build(); + + mongoClient = MongoClients.create(mongoSettings); + database = mongoClient.getDatabase(settings.getDatabaseName()); + collection = database.getCollection(settings.getCollectionName()); + collection.createIndex(Indexes.hashed("session")); + collection.createIndex(Indexes.hashed("dataType")); + } + + @Override + public void disconnect() { + + if (mongoClient == null) { + return; + } + mongoClient.close(); + } + + + @Override + public void insert(Document document) { + collection.insertOne(document); + } +}