diff --git a/extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/api/settings/FileDataCollectorSettings.java b/extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/api/settings/FileDataCollectorSettings.java index 0eee68f..13dec3a 100644 --- a/extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/api/settings/FileDataCollectorSettings.java +++ b/extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/api/settings/FileDataCollectorSettings.java @@ -25,9 +25,14 @@ package io.github.jwdeveloper.tiktok.extension.collector.api.settings; import lombok.Data; import java.io.File; +import java.util.function.*; @Data public class FileDataCollectorSettings { private File parentFile; + private BiPredicate typeFilter = (dataType, dataTypeName) -> true; + private Predicate userFilter = (tiktokUser) -> true; + private boolean useFileLocks = false; + private boolean appendUserName = false; } \ No newline at end of file diff --git a/extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/impl/storages/FileStorage.java b/extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/impl/storages/FileStorage.java index 08ffcaa..05625ec 100644 --- a/extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/impl/storages/FileStorage.java +++ b/extension-collector/src/main/java/io/github/jwdeveloper/tiktok/extension/collector/impl/storages/FileStorage.java @@ -5,17 +5,20 @@ import io.github.jwdeveloper.tiktok.extension.collector.api.settings.FileDataCol import org.bson.Document; import org.bson.json.JsonWriterSettings; -import java.io.File; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.StandardOpenOption; +import java.io.*; +import java.nio.file.*; +import java.util.*; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.locks.*; public class FileStorage implements Storage { private final FileDataCollectorSettings settings; + private final Map locks; public FileStorage(FileDataCollectorSettings fileDataCollectorSettings) { this.settings = fileDataCollectorSettings; + this.locks = settings.isUseFileLocks() ? new ConcurrentHashMap<>() : null; } @Override @@ -30,13 +33,25 @@ public class FileStorage implements Storage { @Override public void insert(Document document) { - var fileName = document.get("dataType") + "_" + document.get("dataTypeName") + ".json"; + if (settings.getTypeFilter().test(document.getString("dataType"), document.getString("dataTypeName")) && settings.getUserFilter().test(document.getString("tiktokUser"))) { + var fileName = document.get("dataType") + "_" + document.get("dataTypeName") + (settings.isAppendUserName() ? document.getString("tiktokUser") : "") + ".json"; + if (settings.isUseFileLocks()) { + var lock = locks.computeIfAbsent(fileName, s -> new ReentrantLock()); + lock.lock(); + save(document, fileName); + lock.unlock(); + } else + save(document, fileName); + } + } + + private void save(Document document, String fileName) { try { var file = new File(settings.getParentFile(), fileName); file.createNewFile(); - Files.writeString(file.toPath(), document.toJson(JsonWriterSettings.builder().indent(true).build()), StandardOpenOption.APPEND); + Files.writeString(file.toPath(), document.toJson(JsonWriterSettings.builder().indent(true).build())+'\n', StandardOpenOption.APPEND); } catch (IOException e) { e.printStackTrace(); } } -} +} \ No newline at end of file