mirror of
https://github.com/jwdeveloper/TikTokLiveJava.git
synced 2026-02-27 16:59:39 -05:00
Removed unused projects.
This commit is contained in:
@@ -40,12 +40,11 @@ public class CollectorExample {
|
|||||||
|
|
||||||
public static void main(String[] args) throws IOException {
|
public static void main(String[] args) throws IOException {
|
||||||
|
|
||||||
var collector = TikTokLiveCollector.use(settings ->
|
var collector = TikTokLiveCollector.useMongo(settings ->
|
||||||
{
|
{
|
||||||
settings.setConnectionUrl("mongodb+srv://" + mongoUser + ":" + mongoPassword + "@" + mongoDatabase + "/?retryWrites=true&w=majority");
|
settings.setConnectionUrl("mongodb+srv://" + mongoUser + ":" + mongoPassword + "@" + mongoDatabase + "/?retryWrites=true&w=majority");
|
||||||
settings.setDatabaseName("tiktok");
|
|
||||||
});
|
});
|
||||||
collector.connectDatabase();
|
collector.connect();
|
||||||
|
|
||||||
var users = List.of("tehila_723", "dino123597", "domaxyzx", "dash4214", "obserwacje_live");
|
var users = List.of("tehila_723", "dino123597", "domaxyzx", "dash4214", "obserwacje_live");
|
||||||
Map<String, Object> additionalDataFields = Map.of("sessionTag", "ExampleTag");
|
Map<String, Object> additionalDataFields = Map.of("sessionTag", "ExampleTag");
|
||||||
@@ -71,6 +70,6 @@ public class CollectorExample {
|
|||||||
}
|
}
|
||||||
|
|
||||||
System.in.read();
|
System.in.read();
|
||||||
collector.disconnectDatabase();
|
collector.disconnect();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -22,25 +22,34 @@
|
|||||||
*/
|
*/
|
||||||
package io.github.jwdeveloper.tiktok.extension.collector;
|
package io.github.jwdeveloper.tiktok.extension.collector;
|
||||||
|
|
||||||
import io.github.jwdeveloper.tiktok.extension.collector.api.file.FileDataCollectorSettings;
|
import io.github.jwdeveloper.tiktok.extension.collector.api.settings.FileDataCollectorSettings;
|
||||||
import io.github.jwdeveloper.tiktok.extension.collector.api.mongo.MongoDataCollectorSettings;
|
import io.github.jwdeveloper.tiktok.extension.collector.api.settings.mongo.MongoDataCollectorSettings;
|
||||||
import io.github.jwdeveloper.tiktok.extension.collector.impl.*;
|
import io.github.jwdeveloper.tiktok.extension.collector.impl.*;
|
||||||
|
import io.github.jwdeveloper.tiktok.extension.collector.impl.storages.FileStorage;
|
||||||
|
import io.github.jwdeveloper.tiktok.extension.collector.impl.storages.MongoStorage;
|
||||||
|
|
||||||
import java.util.function.Consumer;
|
import java.util.function.Consumer;
|
||||||
|
|
||||||
|
|
||||||
|
/**
|
||||||
|
*
|
||||||
|
*/
|
||||||
public class TikTokLiveCollector
|
public class TikTokLiveCollector
|
||||||
{
|
{
|
||||||
public static MongoDataCollector useMongo(Consumer<MongoDataCollectorSettings> consumer)
|
|
||||||
{
|
public static DataCollector useMongo(Consumer<MongoDataCollectorSettings> consumer) {
|
||||||
var settings = new MongoDataCollectorSettings();
|
var settings = new MongoDataCollectorSettings();
|
||||||
consumer.accept(settings);
|
consumer.accept(settings);
|
||||||
return new MongoDataCollector(settings);
|
|
||||||
|
var storage = new MongoStorage(settings);
|
||||||
|
return new DataCollector(storage);
|
||||||
}
|
}
|
||||||
|
|
||||||
public static FileDataCollector useFile(Consumer<FileDataCollectorSettings> consumer)
|
public static DataCollector useFile(Consumer<FileDataCollectorSettings> consumer) {
|
||||||
{
|
|
||||||
var settings = new FileDataCollectorSettings();
|
var settings = new FileDataCollectorSettings();
|
||||||
consumer.accept(settings);
|
consumer.accept(settings);
|
||||||
return new FileDataCollector(settings);
|
|
||||||
|
var storage = new FileStorage(settings);
|
||||||
|
return new DataCollector(storage);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -0,0 +1,11 @@
|
|||||||
|
package io.github.jwdeveloper.tiktok.extension.collector.api;
|
||||||
|
|
||||||
|
import org.bson.Document;
|
||||||
|
|
||||||
|
public interface Storage {
|
||||||
|
void connect();
|
||||||
|
|
||||||
|
void disconnect();
|
||||||
|
|
||||||
|
void insert(Document document);
|
||||||
|
}
|
||||||
@@ -1,6 +1,7 @@
|
|||||||
package io.github.jwdeveloper.tiktok.extension.collector.api.data;
|
package io.github.jwdeveloper.tiktok.extension.collector.api.settings;
|
||||||
|
|
||||||
import lombok.Data;
|
import lombok.Data;
|
||||||
|
import org.bson.Document;
|
||||||
|
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
import java.util.function.Function;
|
import java.util.function.Function;
|
||||||
@@ -8,5 +9,5 @@ import java.util.function.Function;
|
|||||||
@Data
|
@Data
|
||||||
public class CollectorListenerSettings {
|
public class CollectorListenerSettings {
|
||||||
private Map<String, Object> extraFields;
|
private Map<String, Object> extraFields;
|
||||||
private Function<Object, Boolean> filter;
|
private Function<Document, Boolean> filter;
|
||||||
}
|
}
|
||||||
@@ -20,7 +20,7 @@
|
|||||||
* OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
* OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
||||||
* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||||
*/
|
*/
|
||||||
package io.github.jwdeveloper.tiktok.extension.collector.api.file;
|
package io.github.jwdeveloper.tiktok.extension.collector.api.settings;
|
||||||
|
|
||||||
import lombok.Data;
|
import lombok.Data;
|
||||||
|
|
||||||
@@ -20,7 +20,7 @@
|
|||||||
* OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
* OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
||||||
* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||||
*/
|
*/
|
||||||
package io.github.jwdeveloper.tiktok.extension.collector.api.mongo;
|
package io.github.jwdeveloper.tiktok.extension.collector.api.settings.mongo;
|
||||||
|
|
||||||
import lombok.Setter;
|
import lombok.Setter;
|
||||||
import lombok.experimental.Accessors;
|
import lombok.experimental.Accessors;
|
||||||
@@ -20,7 +20,7 @@
|
|||||||
* OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
* OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
||||||
* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||||
*/
|
*/
|
||||||
package io.github.jwdeveloper.tiktok.extension.collector.api.mongo;
|
package io.github.jwdeveloper.tiktok.extension.collector.api.settings.mongo;
|
||||||
|
|
||||||
import lombok.*;
|
import lombok.*;
|
||||||
|
|
||||||
@@ -29,13 +29,14 @@ import java.util.function.Consumer;
|
|||||||
@Data
|
@Data
|
||||||
public class MongoDataCollectorSettings {
|
public class MongoDataCollectorSettings {
|
||||||
|
|
||||||
|
@Setter
|
||||||
private String connectionUrl;
|
private String connectionUrl;
|
||||||
|
|
||||||
private String databaseName;
|
private String databaseName = "tiktok";
|
||||||
|
|
||||||
private String sessionTag;
|
private String collectionName = "data";
|
||||||
|
|
||||||
public void setConnectionUrl(Consumer<MongoDBConnectionStringBuilder> consumer) {
|
public void connectionBuilder(Consumer<MongoDBConnectionStringBuilder> consumer) {
|
||||||
var builder = new MongoDBConnectionStringBuilder();
|
var builder = new MongoDBConnectionStringBuilder();
|
||||||
consumer.accept(builder);
|
consumer.accept(builder);
|
||||||
connectionUrl = builder.build();
|
connectionUrl = builder.build();
|
||||||
@@ -22,33 +22,41 @@
|
|||||||
*/
|
*/
|
||||||
package io.github.jwdeveloper.tiktok.extension.collector.impl;
|
package io.github.jwdeveloper.tiktok.extension.collector.impl;
|
||||||
|
|
||||||
import io.github.jwdeveloper.tiktok.extension.collector.api.data.CollectorListenerSettings;
|
import io.github.jwdeveloper.tiktok.extension.collector.api.Storage;
|
||||||
import io.github.jwdeveloper.tiktok.extension.collector.api.file.FileDataCollectorSettings;
|
import io.github.jwdeveloper.tiktok.extension.collector.api.settings.CollectorListenerSettings;
|
||||||
|
import org.bson.Document;
|
||||||
|
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
import java.util.function.Function;
|
import java.util.function.Function;
|
||||||
|
|
||||||
public class FileDataCollector {
|
public class DataCollector {
|
||||||
|
|
||||||
private final FileDataCollectorSettings settings;
|
private final Storage storage;
|
||||||
|
|
||||||
public FileDataCollector(FileDataCollectorSettings settings) {
|
public DataCollector(Storage storage) {
|
||||||
this.settings = settings;
|
this.storage = storage;
|
||||||
}
|
}
|
||||||
|
|
||||||
public FileDataCollectorListener newListener() {
|
public void connect() {
|
||||||
|
storage.connect();
|
||||||
|
}
|
||||||
|
public void disconnect() {
|
||||||
|
storage.disconnect();
|
||||||
|
}
|
||||||
|
|
||||||
|
public DataCollectorListener newListener() {
|
||||||
return newListener(Map.of());
|
return newListener(Map.of());
|
||||||
}
|
}
|
||||||
|
|
||||||
public FileDataCollectorListener newListener(Map<String, Object> additionalFields) {
|
public DataCollectorListener newListener(Map<String, Object> additionalFields) {
|
||||||
return newListener(additionalFields, (e) -> true);
|
return newListener(additionalFields, (e) -> true);
|
||||||
}
|
}
|
||||||
|
|
||||||
public FileDataCollectorListener newListener(Map<String, Object> additionalFields,
|
public DataCollectorListener newListener(Map<String, Object> additionalFields,
|
||||||
Function<Object, Boolean> filter) {
|
Function<Document, Boolean> filter) {
|
||||||
var settings = new CollectorListenerSettings();
|
var settings = new CollectorListenerSettings();
|
||||||
settings.setExtraFields(additionalFields);
|
settings.setExtraFields(additionalFields);
|
||||||
settings.setFilter(filter);
|
settings.setFilter(filter);
|
||||||
return new FileDataCollectorListener(this.settings, settings);
|
return new DataCollectorListener(storage, settings);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1,6 +1,5 @@
|
|||||||
package io.github.jwdeveloper.tiktok.extension.collector.impl;
|
package io.github.jwdeveloper.tiktok.extension.collector.impl;
|
||||||
|
|
||||||
import com.mongodb.client.MongoCollection;
|
|
||||||
import io.github.jwdeveloper.tiktok.annotations.TikTokEventObserver;
|
import io.github.jwdeveloper.tiktok.annotations.TikTokEventObserver;
|
||||||
import io.github.jwdeveloper.tiktok.data.events.TikTokErrorEvent;
|
import io.github.jwdeveloper.tiktok.data.events.TikTokErrorEvent;
|
||||||
import io.github.jwdeveloper.tiktok.data.events.common.TikTokEvent;
|
import io.github.jwdeveloper.tiktok.data.events.common.TikTokEvent;
|
||||||
@@ -8,7 +7,8 @@ import io.github.jwdeveloper.tiktok.data.events.control.TikTokConnectingEvent;
|
|||||||
import io.github.jwdeveloper.tiktok.data.events.websocket.TikTokWebsocketResponseEvent;
|
import io.github.jwdeveloper.tiktok.data.events.websocket.TikTokWebsocketResponseEvent;
|
||||||
import io.github.jwdeveloper.tiktok.exceptions.TikTokLiveMessageException;
|
import io.github.jwdeveloper.tiktok.exceptions.TikTokLiveMessageException;
|
||||||
import io.github.jwdeveloper.tiktok.extension.collector.api.LiveDataCollector;
|
import io.github.jwdeveloper.tiktok.extension.collector.api.LiveDataCollector;
|
||||||
import io.github.jwdeveloper.tiktok.extension.collector.api.data.CollectorListenerSettings;
|
import io.github.jwdeveloper.tiktok.extension.collector.api.Storage;
|
||||||
|
import io.github.jwdeveloper.tiktok.extension.collector.api.settings.CollectorListenerSettings;
|
||||||
import io.github.jwdeveloper.tiktok.live.LiveClient;
|
import io.github.jwdeveloper.tiktok.live.LiveClient;
|
||||||
import io.github.jwdeveloper.tiktok.messages.webcast.WebcastResponse;
|
import io.github.jwdeveloper.tiktok.messages.webcast.WebcastResponse;
|
||||||
import io.github.jwdeveloper.tiktok.utils.JsonUtil;
|
import io.github.jwdeveloper.tiktok.utils.JsonUtil;
|
||||||
@@ -19,15 +19,15 @@ import java.io.StringWriter;
|
|||||||
import java.util.Base64;
|
import java.util.Base64;
|
||||||
import java.util.UUID;
|
import java.util.UUID;
|
||||||
|
|
||||||
public class MongoDataCollectorListener implements LiveDataCollector {
|
public class DataCollectorListener implements LiveDataCollector {
|
||||||
|
|
||||||
private final MongoCollection<Document> collection;
|
private final Storage storage;
|
||||||
private final CollectorListenerSettings settings;
|
private final CollectorListenerSettings settings;
|
||||||
private String sessionId;
|
private String sessionId;
|
||||||
private String userName;
|
private String userName;
|
||||||
|
|
||||||
public MongoDataCollectorListener(MongoCollection<Document> collection, CollectorListenerSettings settings) {
|
public DataCollectorListener(Storage collection, CollectorListenerSettings settings) {
|
||||||
this.collection = collection;
|
this.storage = collection;
|
||||||
this.settings = settings;
|
this.settings = settings;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -100,7 +100,7 @@ public class MongoDataCollectorListener implements LiveDataCollector {
|
|||||||
if (!settings.getFilter().apply(document)) {
|
if (!settings.getFilter().apply(document)) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
collection.insertOne(document);
|
storage.insert(document);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@@ -1,124 +0,0 @@
|
|||||||
package io.github.jwdeveloper.tiktok.extension.collector.impl;
|
|
||||||
|
|
||||||
import com.google.gson.*;
|
|
||||||
import io.github.jwdeveloper.tiktok.annotations.TikTokEventObserver;
|
|
||||||
import io.github.jwdeveloper.tiktok.data.events.TikTokErrorEvent;
|
|
||||||
import io.github.jwdeveloper.tiktok.data.events.common.TikTokEvent;
|
|
||||||
import io.github.jwdeveloper.tiktok.data.events.control.TikTokConnectingEvent;
|
|
||||||
import io.github.jwdeveloper.tiktok.data.events.websocket.TikTokWebsocketResponseEvent;
|
|
||||||
import io.github.jwdeveloper.tiktok.exceptions.TikTokLiveMessageException;
|
|
||||||
import io.github.jwdeveloper.tiktok.extension.collector.api.LiveDataCollector;
|
|
||||||
import io.github.jwdeveloper.tiktok.extension.collector.api.data.CollectorListenerSettings;
|
|
||||||
import io.github.jwdeveloper.tiktok.extension.collector.api.file.FileDataCollectorSettings;
|
|
||||||
import io.github.jwdeveloper.tiktok.live.LiveClient;
|
|
||||||
import io.github.jwdeveloper.tiktok.messages.webcast.WebcastResponse;
|
|
||||||
import io.github.jwdeveloper.tiktok.utils.JsonUtil;
|
|
||||||
|
|
||||||
import java.io.*;
|
|
||||||
import java.nio.file.*;
|
|
||||||
import java.util.*;
|
|
||||||
|
|
||||||
public class FileDataCollectorListener implements LiveDataCollector {
|
|
||||||
|
|
||||||
private final FileDataCollectorSettings fileSettings;
|
|
||||||
private final CollectorListenerSettings collectorSettings;
|
|
||||||
private String sessionId;
|
|
||||||
private String userName;
|
|
||||||
|
|
||||||
public FileDataCollectorListener(FileDataCollectorSettings fileSettings, CollectorListenerSettings collectorSettings) {
|
|
||||||
this.fileSettings = fileSettings;
|
|
||||||
this.collectorSettings = collectorSettings;
|
|
||||||
}
|
|
||||||
|
|
||||||
@TikTokEventObserver
|
|
||||||
private void onResponse(LiveClient liveClient, TikTokWebsocketResponseEvent event) {
|
|
||||||
includeResponse(liveClient, event.getResponse());
|
|
||||||
event.getResponse().getMessagesList().forEach(message -> includeMessage(liveClient, message));
|
|
||||||
}
|
|
||||||
|
|
||||||
@TikTokEventObserver
|
|
||||||
private void onEvent(LiveClient liveClient, TikTokEvent event) {
|
|
||||||
if (event instanceof TikTokConnectingEvent) {
|
|
||||||
sessionId = UUID.randomUUID().toString();
|
|
||||||
userName = liveClient.getRoomInfo().getHostName();
|
|
||||||
}
|
|
||||||
|
|
||||||
if (event instanceof TikTokErrorEvent) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
includeEvent(event);
|
|
||||||
}
|
|
||||||
|
|
||||||
@TikTokEventObserver
|
|
||||||
private void onError(LiveClient liveClient, TikTokErrorEvent event) {
|
|
||||||
event.getException().printStackTrace();
|
|
||||||
includeError(event);
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
private void includeResponse(LiveClient liveClient, WebcastResponse message) {
|
|
||||||
var messageContent = Base64.getEncoder().encodeToString(message.toByteArray());
|
|
||||||
saveJson(createJson("response", "webcast", messageContent));
|
|
||||||
}
|
|
||||||
|
|
||||||
private void includeMessage(LiveClient liveClient, WebcastResponse.Message message) {
|
|
||||||
var method = message.getMethod();
|
|
||||||
var messageContent = Base64.getEncoder().encodeToString(message.getPayload().toByteArray());
|
|
||||||
|
|
||||||
saveJson(createJson("message", method, messageContent));
|
|
||||||
}
|
|
||||||
|
|
||||||
private void includeEvent(TikTokEvent event) {
|
|
||||||
var json = JsonUtil.toJson(event);
|
|
||||||
var content = Base64.getEncoder().encodeToString(json.getBytes());
|
|
||||||
var name = event.getClass().getSimpleName();
|
|
||||||
saveJson(createJson("event", name, content));
|
|
||||||
}
|
|
||||||
|
|
||||||
private void includeError(TikTokErrorEvent event) {
|
|
||||||
var exception = event.getException();
|
|
||||||
var exceptionName = event.getException().getClass().getSimpleName();
|
|
||||||
|
|
||||||
var sw = new StringWriter();
|
|
||||||
var pw = new PrintWriter(sw);
|
|
||||||
event.getException().printStackTrace(pw);
|
|
||||||
var content = sw.toString();
|
|
||||||
|
|
||||||
var json = createJson("error", exceptionName, content);
|
|
||||||
if (exception instanceof TikTokLiveMessageException ex) {
|
|
||||||
json.addProperty("message", ex.messageToBase64());
|
|
||||||
json.addProperty("response", ex.webcastResponseToBase64());
|
|
||||||
}
|
|
||||||
saveJson(json);
|
|
||||||
}
|
|
||||||
|
|
||||||
private void saveJson(JsonObject jsonObject) {
|
|
||||||
if (!collectorSettings.getFilter().apply(jsonObject)) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
try {
|
|
||||||
File file = new File(fileSettings.getParentFile(), jsonObject.get("dataType").getAsString()+":"+jsonObject.get("dataTypeName").getAsString()+".txt");
|
|
||||||
file.createNewFile();
|
|
||||||
Files.writeString(file.toPath(), jsonObject.toString(), StandardOpenOption.APPEND);
|
|
||||||
} catch (IOException e) {
|
|
||||||
e.printStackTrace();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private JsonObject createJson(String dataType, String dataTypeName, String content) {
|
|
||||||
JsonObject data = new JsonObject();
|
|
||||||
data.addProperty("session", sessionId);
|
|
||||||
for (var entry : collectorSettings.getExtraFields().entrySet()) {
|
|
||||||
if (entry.getValue() instanceof JsonElement element)
|
|
||||||
data.add(entry.getKey(), element);
|
|
||||||
else
|
|
||||||
data.addProperty(entry.getKey(), entry.getValue().toString());
|
|
||||||
}
|
|
||||||
data.addProperty("tiktokUser", userName);
|
|
||||||
data.addProperty("dataType", dataType);
|
|
||||||
data.addProperty("dataTypeName", dataTypeName);
|
|
||||||
data.addProperty("content", content);
|
|
||||||
return data;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,88 +0,0 @@
|
|||||||
/*
|
|
||||||
* Copyright (c) 2023-2023 jwdeveloper jacekwoln@gmail.com
|
|
||||||
*
|
|
||||||
* Permission is hereby granted, free of charge, to any person obtaining
|
|
||||||
* a copy of this software and associated documentation files (the
|
|
||||||
* "Software"), to deal in the Software without restriction, including
|
|
||||||
* without limitation the rights to use, copy, modify, merge, publish,
|
|
||||||
* distribute, sublicense, and/or sell copies of the Software, and to
|
|
||||||
* permit persons to whom the Software is furnished to do so, subject to
|
|
||||||
* the following conditions:
|
|
||||||
*
|
|
||||||
* The above copyright notice and this permission notice shall be
|
|
||||||
* included in all copies or substantial portions of the Software.
|
|
||||||
*
|
|
||||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
|
||||||
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
|
||||||
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
|
||||||
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
|
||||||
* LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
|
||||||
* OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
|
||||||
* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
|
||||||
*/
|
|
||||||
package io.github.jwdeveloper.tiktok.extension.collector.impl;
|
|
||||||
|
|
||||||
import com.mongodb.ConnectionString;
|
|
||||||
import com.mongodb.MongoClientSettings;
|
|
||||||
import com.mongodb.ServerApi;
|
|
||||||
import com.mongodb.ServerApiVersion;
|
|
||||||
import com.mongodb.client.MongoClient;
|
|
||||||
import com.mongodb.client.MongoClients;
|
|
||||||
import com.mongodb.client.MongoCollection;
|
|
||||||
import com.mongodb.client.MongoDatabase;
|
|
||||||
import com.mongodb.client.model.Indexes;
|
|
||||||
import io.github.jwdeveloper.tiktok.extension.collector.api.data.CollectorListenerSettings;
|
|
||||||
import io.github.jwdeveloper.tiktok.extension.collector.api.mongo.MongoDataCollectorSettings;
|
|
||||||
import org.bson.Document;
|
|
||||||
|
|
||||||
import java.util.Map;
|
|
||||||
import java.util.function.Function;
|
|
||||||
|
|
||||||
public class MongoDataCollector {
|
|
||||||
|
|
||||||
private final MongoDataCollectorSettings settings;
|
|
||||||
private MongoClient mongoClient;
|
|
||||||
private MongoDatabase database;
|
|
||||||
private MongoCollection<Document> collection;
|
|
||||||
|
|
||||||
public MongoDataCollector(MongoDataCollectorSettings settings) {
|
|
||||||
this.settings = settings;
|
|
||||||
}
|
|
||||||
|
|
||||||
public void connectDatabase() {
|
|
||||||
var serverApi = ServerApi.builder()
|
|
||||||
.version(ServerApiVersion.V1)
|
|
||||||
.build();
|
|
||||||
var mongoSettings = MongoClientSettings.builder()
|
|
||||||
.applyConnectionString(new ConnectionString(settings.getConnectionUrl()))
|
|
||||||
.serverApi(serverApi)
|
|
||||||
.build();
|
|
||||||
|
|
||||||
mongoClient = MongoClients.create(mongoSettings);
|
|
||||||
database = mongoClient.getDatabase(settings.getDatabaseName());
|
|
||||||
collection = database.getCollection("data");
|
|
||||||
collection.createIndex(Indexes.hashed("session"));
|
|
||||||
collection.createIndex(Indexes.hashed("dataType"));
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
public void disconnectDatabase() {
|
|
||||||
mongoClient.close();
|
|
||||||
}
|
|
||||||
|
|
||||||
public MongoDataCollectorListener newListener() {
|
|
||||||
return newListener(Map.of());
|
|
||||||
}
|
|
||||||
|
|
||||||
public MongoDataCollectorListener newListener(Map<String, Object> additionalFields) {
|
|
||||||
return newListener(additionalFields, (e)->true);
|
|
||||||
}
|
|
||||||
|
|
||||||
public MongoDataCollectorListener newListener(Map<String, Object> additionalFields,
|
|
||||||
Function<Object, Boolean> filter) {
|
|
||||||
var settings = new CollectorListenerSettings();
|
|
||||||
settings.setExtraFields(additionalFields);
|
|
||||||
settings.setFilter(filter);
|
|
||||||
return new MongoDataCollectorListener(collection, settings);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -0,0 +1,41 @@
|
|||||||
|
package io.github.jwdeveloper.tiktok.extension.collector.impl.storages;
|
||||||
|
|
||||||
|
import io.github.jwdeveloper.tiktok.extension.collector.api.Storage;
|
||||||
|
import io.github.jwdeveloper.tiktok.extension.collector.api.settings.FileDataCollectorSettings;
|
||||||
|
import org.bson.Document;
|
||||||
|
|
||||||
|
import java.io.File;
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.nio.file.Files;
|
||||||
|
import java.nio.file.StandardOpenOption;
|
||||||
|
|
||||||
|
public class FileStorage implements Storage {
|
||||||
|
|
||||||
|
private final FileDataCollectorSettings settings;
|
||||||
|
|
||||||
|
public FileStorage(FileDataCollectorSettings fileDataCollectorSettings) {
|
||||||
|
this.settings = fileDataCollectorSettings;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void connect() {
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void disconnect() {
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void insert(Document document) {
|
||||||
|
var fileName = document.get("dataType") + ":" + document.get("dataTypeName") + ".json";
|
||||||
|
try {
|
||||||
|
var file = new File(settings.getParentFile(), fileName);
|
||||||
|
file.createNewFile();
|
||||||
|
Files.writeString(file.toPath(), document.toJson(), StandardOpenOption.APPEND);
|
||||||
|
} catch (IOException e) {
|
||||||
|
e.printStackTrace();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,58 @@
|
|||||||
|
package io.github.jwdeveloper.tiktok.extension.collector.impl.storages;
|
||||||
|
|
||||||
|
import com.mongodb.ConnectionString;
|
||||||
|
import com.mongodb.MongoClientSettings;
|
||||||
|
import com.mongodb.ServerApi;
|
||||||
|
import com.mongodb.ServerApiVersion;
|
||||||
|
import com.mongodb.client.MongoClient;
|
||||||
|
import com.mongodb.client.MongoClients;
|
||||||
|
import com.mongodb.client.MongoCollection;
|
||||||
|
import com.mongodb.client.MongoDatabase;
|
||||||
|
import com.mongodb.client.model.Indexes;
|
||||||
|
import io.github.jwdeveloper.tiktok.extension.collector.api.Storage;
|
||||||
|
import io.github.jwdeveloper.tiktok.extension.collector.api.settings.mongo.MongoDataCollectorSettings;
|
||||||
|
import org.bson.Document;
|
||||||
|
|
||||||
|
public class MongoStorage implements Storage {
|
||||||
|
private MongoClient mongoClient;
|
||||||
|
private MongoDatabase database;
|
||||||
|
private MongoCollection<Document> collection;
|
||||||
|
private final MongoDataCollectorSettings settings;
|
||||||
|
|
||||||
|
public MongoStorage(MongoDataCollectorSettings settings) {
|
||||||
|
this.settings = settings;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void connect() {
|
||||||
|
|
||||||
|
var serverApi = ServerApi.builder()
|
||||||
|
.version(ServerApiVersion.V1)
|
||||||
|
.build();
|
||||||
|
var mongoSettings = MongoClientSettings.builder()
|
||||||
|
.applyConnectionString(new ConnectionString(settings.getConnectionUrl()))
|
||||||
|
.serverApi(serverApi)
|
||||||
|
.build();
|
||||||
|
|
||||||
|
mongoClient = MongoClients.create(mongoSettings);
|
||||||
|
database = mongoClient.getDatabase(settings.getDatabaseName());
|
||||||
|
collection = database.getCollection(settings.getCollectionName());
|
||||||
|
collection.createIndex(Indexes.hashed("session"));
|
||||||
|
collection.createIndex(Indexes.hashed("dataType"));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void disconnect() {
|
||||||
|
|
||||||
|
if (mongoClient == null) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
mongoClient.close();
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void insert(Document document) {
|
||||||
|
collection.insertOne(document);
|
||||||
|
}
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user