Merge remote-tracking branch 'origin/develop-1.5.0' into develop-1.5.0-live-user-data-fix

# Conflicts:
#	extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/api/settings/mongo/MongoDataCollectorSettings.java
This commit is contained in:
kohlerpop1
2024-02-29 20:23:32 -05:00
13 changed files with 166 additions and 250 deletions

View File

@@ -44,9 +44,8 @@ public class CollectorExample {
var collector = TikTokLiveCollector.useMongo(settings -> var collector = TikTokLiveCollector.useMongo(settings ->
{ {
settings.setConnectionUrl("mongodb+srv://" + mongoUser + ":" + mongoPassword + "@" + mongoDatabase + "/?retryWrites=true&w=majority"); settings.setConnectionUrl("mongodb+srv://" + mongoUser + ":" + mongoPassword + "@" + mongoDatabase + "/?retryWrites=true&w=majority");
settings.setDatabaseName("tiktok");
}); });
collector.connectDatabase(); collector.connect();
var users = List.of("tehila_723", "dino123597", "domaxyzx", "dash4214", "obserwacje_live"); var users = List.of("tehila_723", "dino123597", "domaxyzx", "dash4214", "obserwacje_live");
Map<String, Object> additionalDataFields = Map.of("sessionTag", "ExampleTag"); Map<String, Object> additionalDataFields = Map.of("sessionTag", "ExampleTag");
@@ -72,6 +71,6 @@ public class CollectorExample {
} }
System.in.read(); System.in.read();
collector.disconnectDatabase(); collector.disconnect();
} }
} }

View File

@@ -22,25 +22,34 @@
*/ */
package io.github.jwdeveloper.tiktok.extension.collector; package io.github.jwdeveloper.tiktok.extension.collector;
import io.github.jwdeveloper.tiktok.extension.collector.api.file.FileDataCollectorSettings; import io.github.jwdeveloper.tiktok.extension.collector.api.settings.FileDataCollectorSettings;
import io.github.jwdeveloper.tiktok.extension.collector.api.mongo.MongoDataCollectorSettings; import io.github.jwdeveloper.tiktok.extension.collector.api.settings.mongo.MongoDataCollectorSettings;
import io.github.jwdeveloper.tiktok.extension.collector.impl.*; import io.github.jwdeveloper.tiktok.extension.collector.impl.*;
import io.github.jwdeveloper.tiktok.extension.collector.impl.storages.FileStorage;
import io.github.jwdeveloper.tiktok.extension.collector.impl.storages.MongoStorage;
import java.util.function.Consumer; import java.util.function.Consumer;
/**
*
*/
public class TikTokLiveCollector public class TikTokLiveCollector
{ {
public static MongoDataCollector useMongo(Consumer<MongoDataCollectorSettings> consumer)
{ public static DataCollector useMongo(Consumer<MongoDataCollectorSettings> consumer) {
var settings = new MongoDataCollectorSettings(); var settings = new MongoDataCollectorSettings();
consumer.accept(settings); consumer.accept(settings);
return new MongoDataCollector(settings);
var storage = new MongoStorage(settings);
return new DataCollector(storage);
} }
public static FileDataCollector useFile(Consumer<FileDataCollectorSettings> consumer) public static DataCollector useFile(Consumer<FileDataCollectorSettings> consumer) {
{
var settings = new FileDataCollectorSettings(); var settings = new FileDataCollectorSettings();
consumer.accept(settings); consumer.accept(settings);
return new FileDataCollector(settings);
var storage = new FileStorage(settings);
return new DataCollector(storage);
} }
} }

View File

@@ -0,0 +1,11 @@
package io.github.jwdeveloper.tiktok.extension.collector.api;
import org.bson.Document;
public interface Storage {
void connect();
void disconnect();
void insert(Document document);
}

View File

@@ -1,6 +1,7 @@
package io.github.jwdeveloper.tiktok.extension.collector.api.data; package io.github.jwdeveloper.tiktok.extension.collector.api.settings;
import lombok.Data; import lombok.Data;
import org.bson.Document;
import java.util.Map; import java.util.Map;
import java.util.function.Function; import java.util.function.Function;
@@ -8,5 +9,5 @@ import java.util.function.Function;
@Data @Data
public class CollectorListenerSettings { public class CollectorListenerSettings {
private Map<String, Object> extraFields; private Map<String, Object> extraFields;
private Function<Object, Boolean> filter; private Function<Document, Boolean> filter;
} }

View File

@@ -20,7 +20,7 @@
* OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/ */
package io.github.jwdeveloper.tiktok.extension.collector.api.file; package io.github.jwdeveloper.tiktok.extension.collector.api.settings;
import lombok.Data; import lombok.Data;

View File

@@ -20,7 +20,7 @@
* OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/ */
package io.github.jwdeveloper.tiktok.extension.collector.api.mongo; package io.github.jwdeveloper.tiktok.extension.collector.api.settings.mongo;
import lombok.Setter; import lombok.Setter;
import lombok.experimental.Accessors; import lombok.experimental.Accessors;

View File

@@ -20,7 +20,7 @@
* OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/ */
package io.github.jwdeveloper.tiktok.extension.collector.api.mongo; package io.github.jwdeveloper.tiktok.extension.collector.api.settings.mongo;
import lombok.*; import lombok.*;
@@ -29,13 +29,14 @@ import java.util.function.Consumer;
@Data @Data
public class MongoDataCollectorSettings { public class MongoDataCollectorSettings {
@Setter
private String connectionUrl; private String connectionUrl;
private String databaseName; private String databaseName = "tiktok";
private String sessionTag; private String collectionName = "data";
public void setConnectionUrlConsumer(Consumer<MongoDBConnectionStringBuilder> consumer) { public void connectionBuilder(Consumer<MongoDBConnectionStringBuilder> consumer) {
var builder = new MongoDBConnectionStringBuilder(); var builder = new MongoDBConnectionStringBuilder();
consumer.accept(builder); consumer.accept(builder);
connectionUrl = builder.build(); connectionUrl = builder.build();

View File

@@ -22,33 +22,41 @@
*/ */
package io.github.jwdeveloper.tiktok.extension.collector.impl; package io.github.jwdeveloper.tiktok.extension.collector.impl;
import io.github.jwdeveloper.tiktok.extension.collector.api.data.CollectorListenerSettings; import io.github.jwdeveloper.tiktok.extension.collector.api.Storage;
import io.github.jwdeveloper.tiktok.extension.collector.api.file.FileDataCollectorSettings; import io.github.jwdeveloper.tiktok.extension.collector.api.settings.CollectorListenerSettings;
import org.bson.Document;
import java.util.Map; import java.util.Map;
import java.util.function.Function; import java.util.function.Function;
public class FileDataCollector { public class DataCollector {
private final FileDataCollectorSettings settings; private final Storage storage;
public FileDataCollector(FileDataCollectorSettings settings) { public DataCollector(Storage storage) {
this.settings = settings; this.storage = storage;
} }
public FileDataCollectorListener newListener() { public void connect() {
storage.connect();
}
public void disconnect() {
storage.disconnect();
}
public DataCollectorListener newListener() {
return newListener(Map.of()); return newListener(Map.of());
} }
public FileDataCollectorListener newListener(Map<String, Object> additionalFields) { public DataCollectorListener newListener(Map<String, Object> additionalFields) {
return newListener(additionalFields, (e)->true); return newListener(additionalFields, (e) -> true);
} }
public FileDataCollectorListener newListener(Map<String, Object> additionalFields, public DataCollectorListener newListener(Map<String, Object> additionalFields,
Function<Object, Boolean> filter) { Function<Document, Boolean> filter) {
var settings = new CollectorListenerSettings(); var settings = new CollectorListenerSettings();
settings.setExtraFields(additionalFields); settings.setExtraFields(additionalFields);
settings.setFilter(filter); settings.setFilter(filter);
return new FileDataCollectorListener(this.settings, settings); return new DataCollectorListener(storage, settings);
} }
} }

View File

@@ -1,6 +1,5 @@
package io.github.jwdeveloper.tiktok.extension.collector.impl; package io.github.jwdeveloper.tiktok.extension.collector.impl;
import com.mongodb.client.MongoCollection;
import io.github.jwdeveloper.tiktok.annotations.TikTokEventObserver; import io.github.jwdeveloper.tiktok.annotations.TikTokEventObserver;
import io.github.jwdeveloper.tiktok.data.events.TikTokErrorEvent; import io.github.jwdeveloper.tiktok.data.events.TikTokErrorEvent;
import io.github.jwdeveloper.tiktok.data.events.common.TikTokEvent; import io.github.jwdeveloper.tiktok.data.events.common.TikTokEvent;
@@ -8,7 +7,8 @@ import io.github.jwdeveloper.tiktok.data.events.control.TikTokConnectingEvent;
import io.github.jwdeveloper.tiktok.data.events.websocket.TikTokWebsocketResponseEvent; import io.github.jwdeveloper.tiktok.data.events.websocket.TikTokWebsocketResponseEvent;
import io.github.jwdeveloper.tiktok.exceptions.TikTokLiveMessageException; import io.github.jwdeveloper.tiktok.exceptions.TikTokLiveMessageException;
import io.github.jwdeveloper.tiktok.extension.collector.api.LiveDataCollector; import io.github.jwdeveloper.tiktok.extension.collector.api.LiveDataCollector;
import io.github.jwdeveloper.tiktok.extension.collector.api.data.CollectorListenerSettings; import io.github.jwdeveloper.tiktok.extension.collector.api.Storage;
import io.github.jwdeveloper.tiktok.extension.collector.api.settings.CollectorListenerSettings;
import io.github.jwdeveloper.tiktok.live.LiveClient; import io.github.jwdeveloper.tiktok.live.LiveClient;
import io.github.jwdeveloper.tiktok.messages.webcast.WebcastResponse; import io.github.jwdeveloper.tiktok.messages.webcast.WebcastResponse;
import io.github.jwdeveloper.tiktok.utils.JsonUtil; import io.github.jwdeveloper.tiktok.utils.JsonUtil;
@@ -19,15 +19,15 @@ import java.io.StringWriter;
import java.util.Base64; import java.util.Base64;
import java.util.UUID; import java.util.UUID;
public class MongoDataCollectorListener implements LiveDataCollector { public class DataCollectorListener implements LiveDataCollector {
private final MongoCollection<Document> collection; private final Storage storage;
private final CollectorListenerSettings settings; private final CollectorListenerSettings settings;
private String sessionId; private String sessionId;
private String userName; private String userName;
public MongoDataCollectorListener(MongoCollection<Document> collection, CollectorListenerSettings settings) { public DataCollectorListener(Storage collection, CollectorListenerSettings settings) {
this.collection = collection; this.storage = collection;
this.settings = settings; this.settings = settings;
} }
@@ -100,7 +100,7 @@ public class MongoDataCollectorListener implements LiveDataCollector {
if (!settings.getFilter().apply(document)) { if (!settings.getFilter().apply(document)) {
return; return;
} }
collection.insertOne(document); storage.insert(document);
} }

View File

@@ -1,124 +0,0 @@
package io.github.jwdeveloper.tiktok.extension.collector.impl;
import com.google.gson.*;
import io.github.jwdeveloper.tiktok.annotations.TikTokEventObserver;
import io.github.jwdeveloper.tiktok.data.events.TikTokErrorEvent;
import io.github.jwdeveloper.tiktok.data.events.common.TikTokEvent;
import io.github.jwdeveloper.tiktok.data.events.control.TikTokConnectingEvent;
import io.github.jwdeveloper.tiktok.data.events.websocket.TikTokWebsocketResponseEvent;
import io.github.jwdeveloper.tiktok.exceptions.TikTokLiveMessageException;
import io.github.jwdeveloper.tiktok.extension.collector.api.LiveDataCollector;
import io.github.jwdeveloper.tiktok.extension.collector.api.data.CollectorListenerSettings;
import io.github.jwdeveloper.tiktok.extension.collector.api.file.FileDataCollectorSettings;
import io.github.jwdeveloper.tiktok.live.LiveClient;
import io.github.jwdeveloper.tiktok.messages.webcast.WebcastResponse;
import io.github.jwdeveloper.tiktok.utils.JsonUtil;
import java.io.*;
import java.nio.file.*;
import java.util.*;
public class FileDataCollectorListener implements LiveDataCollector {
private final FileDataCollectorSettings fileSettings;
private final CollectorListenerSettings collectorSettings;
private String sessionId;
private String userName;
public FileDataCollectorListener(FileDataCollectorSettings fileSettings, CollectorListenerSettings collectorSettings) {
this.fileSettings = fileSettings;
this.collectorSettings = collectorSettings;
}
@TikTokEventObserver
private void onResponse(LiveClient liveClient, TikTokWebsocketResponseEvent event) {
includeResponse(liveClient, event.getResponse());
event.getResponse().getMessagesList().forEach(message -> includeMessage(liveClient, message));
}
@TikTokEventObserver
private void onEvent(LiveClient liveClient, TikTokEvent event) {
if (event instanceof TikTokConnectingEvent) {
sessionId = UUID.randomUUID().toString();
userName = liveClient.getRoomInfo().getHostName();
}
if (event instanceof TikTokErrorEvent) {
return;
}
includeEvent(event);
}
@TikTokEventObserver
private void onError(LiveClient liveClient, TikTokErrorEvent event) {
event.getException().printStackTrace();
includeError(event);
}
private void includeResponse(LiveClient liveClient, WebcastResponse message) {
var messageContent = Base64.getEncoder().encodeToString(message.toByteArray());
saveJson(createJson("response", "webcast", messageContent));
}
private void includeMessage(LiveClient liveClient, WebcastResponse.Message message) {
var method = message.getMethod();
var messageContent = Base64.getEncoder().encodeToString(message.getPayload().toByteArray());
saveJson(createJson("message", method, messageContent));
}
private void includeEvent(TikTokEvent event) {
var json = JsonUtil.toJson(event);
var content = Base64.getEncoder().encodeToString(json.getBytes());
var name = event.getClass().getSimpleName();
saveJson(createJson("event", name, content));
}
private void includeError(TikTokErrorEvent event) {
var exception = event.getException();
var exceptionName = event.getException().getClass().getSimpleName();
var sw = new StringWriter();
var pw = new PrintWriter(sw);
event.getException().printStackTrace(pw);
var content = sw.toString();
var json = createJson("error", exceptionName, content);
if (exception instanceof TikTokLiveMessageException ex) {
json.addProperty("message", ex.messageToBase64());
json.addProperty("response", ex.webcastResponseToBase64());
}
saveJson(json);
}
private void saveJson(JsonObject jsonObject) {
if (!collectorSettings.getFilter().apply(jsonObject)) {
return;
}
try {
File file = new File(fileSettings.getParentFile(), jsonObject.get("dataType").getAsString()+":"+jsonObject.get("dataTypeName").getAsString()+".txt");
file.createNewFile();
Files.writeString(file.toPath(), jsonObject.toString(), StandardOpenOption.APPEND);
} catch (IOException e) {
e.printStackTrace();
}
}
private JsonObject createJson(String dataType, String dataTypeName, String content) {
JsonObject data = new JsonObject();
data.addProperty("session", sessionId);
for (var entry : collectorSettings.getExtraFields().entrySet()) {
if (entry.getValue() instanceof JsonElement element)
data.add(entry.getKey(), element);
else
data.addProperty(entry.getKey(), entry.getValue().toString());
}
data.addProperty("tiktokUser", userName);
data.addProperty("dataType", dataType);
data.addProperty("dataTypeName", dataTypeName);
data.addProperty("content", content);
return data;
}
}

View File

@@ -1,88 +0,0 @@
/*
* Copyright (c) 2023-2023 jwdeveloper jacekwoln@gmail.com
*
* Permission is hereby granted, free of charge, to any person obtaining
* a copy of this software and associated documentation files (the
* "Software"), to deal in the Software without restriction, including
* without limitation the rights to use, copy, modify, merge, publish,
* distribute, sublicense, and/or sell copies of the Software, and to
* permit persons to whom the Software is furnished to do so, subject to
* the following conditions:
*
* The above copyright notice and this permission notice shall be
* included in all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
* LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
* OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
package io.github.jwdeveloper.tiktok.extension.collector.impl;
import com.mongodb.ConnectionString;
import com.mongodb.MongoClientSettings;
import com.mongodb.ServerApi;
import com.mongodb.ServerApiVersion;
import com.mongodb.client.MongoClient;
import com.mongodb.client.MongoClients;
import com.mongodb.client.MongoCollection;
import com.mongodb.client.MongoDatabase;
import com.mongodb.client.model.Indexes;
import io.github.jwdeveloper.tiktok.extension.collector.api.data.CollectorListenerSettings;
import io.github.jwdeveloper.tiktok.extension.collector.api.mongo.MongoDataCollectorSettings;
import org.bson.Document;
import java.util.Map;
import java.util.function.Function;
public class MongoDataCollector {
private final MongoDataCollectorSettings settings;
private MongoClient mongoClient;
private MongoDatabase database;
private MongoCollection<Document> collection;
public MongoDataCollector(MongoDataCollectorSettings settings) {
this.settings = settings;
}
public void connectDatabase() {
var serverApi = ServerApi.builder()
.version(ServerApiVersion.V1)
.build();
var mongoSettings = MongoClientSettings.builder()
.applyConnectionString(new ConnectionString(settings.getConnectionUrl()))
.serverApi(serverApi)
.build();
mongoClient = MongoClients.create(mongoSettings);
database = mongoClient.getDatabase(settings.getDatabaseName());
collection = database.getCollection("data");
collection.createIndex(Indexes.hashed("session"));
collection.createIndex(Indexes.hashed("dataType"));
}
public void disconnectDatabase() {
mongoClient.close();
}
public MongoDataCollectorListener newListener() {
return newListener(Map.of());
}
public MongoDataCollectorListener newListener(Map<String, Object> additionalFields) {
return newListener(additionalFields, (e)->true);
}
public MongoDataCollectorListener newListener(Map<String, Object> additionalFields,
Function<Object, Boolean> filter) {
var settings = new CollectorListenerSettings();
settings.setExtraFields(additionalFields);
settings.setFilter(filter);
return new MongoDataCollectorListener(collection, settings);
}
}

View File

@@ -0,0 +1,41 @@
package io.github.jwdeveloper.tiktok.extension.collector.impl.storages;
import io.github.jwdeveloper.tiktok.extension.collector.api.Storage;
import io.github.jwdeveloper.tiktok.extension.collector.api.settings.FileDataCollectorSettings;
import org.bson.Document;
import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.StandardOpenOption;
public class FileStorage implements Storage {
private final FileDataCollectorSettings settings;
public FileStorage(FileDataCollectorSettings fileDataCollectorSettings) {
this.settings = fileDataCollectorSettings;
}
@Override
public void connect() {
}
@Override
public void disconnect() {
}
@Override
public void insert(Document document) {
var fileName = document.get("dataType") + ":" + document.get("dataTypeName") + ".json";
try {
var file = new File(settings.getParentFile(), fileName);
file.createNewFile();
Files.writeString(file.toPath(), document.toJson(), StandardOpenOption.APPEND);
} catch (IOException e) {
e.printStackTrace();
}
}
}

View File

@@ -0,0 +1,58 @@
package io.github.jwdeveloper.tiktok.extension.collector.impl.storages;
import com.mongodb.ConnectionString;
import com.mongodb.MongoClientSettings;
import com.mongodb.ServerApi;
import com.mongodb.ServerApiVersion;
import com.mongodb.client.MongoClient;
import com.mongodb.client.MongoClients;
import com.mongodb.client.MongoCollection;
import com.mongodb.client.MongoDatabase;
import com.mongodb.client.model.Indexes;
import io.github.jwdeveloper.tiktok.extension.collector.api.Storage;
import io.github.jwdeveloper.tiktok.extension.collector.api.settings.mongo.MongoDataCollectorSettings;
import org.bson.Document;
public class MongoStorage implements Storage {
private MongoClient mongoClient;
private MongoDatabase database;
private MongoCollection<Document> collection;
private final MongoDataCollectorSettings settings;
public MongoStorage(MongoDataCollectorSettings settings) {
this.settings = settings;
}
@Override
public void connect() {
var serverApi = ServerApi.builder()
.version(ServerApiVersion.V1)
.build();
var mongoSettings = MongoClientSettings.builder()
.applyConnectionString(new ConnectionString(settings.getConnectionUrl()))
.serverApi(serverApi)
.build();
mongoClient = MongoClients.create(mongoSettings);
database = mongoClient.getDatabase(settings.getDatabaseName());
collection = database.getCollection(settings.getCollectionName());
collection.createIndex(Indexes.hashed("session"));
collection.createIndex(Indexes.hashed("dataType"));
}
@Override
public void disconnect() {
if (mongoClient == null) {
return;
}
mongoClient.close();
}
@Override
public void insert(Document document) {
collection.insertOne(document);
}
}