update configurationmanager to support configuration file name

src/main/java/de/mediathekview/mserver/base/config/Log4JConfigurationFactory.java

@@ -45,9 +45,19 @@ public class Log4JConfigurationFactory extends ConfigurationFactory {
 
   private static MServerLogSettingsDTO logSettings;
 
+  public Log4JConfigurationFactory(MServerLogSettingsDTO logSettings) {
+    Log4JConfigurationFactory.logSettings = logSettings;
+  }
+
+
+  public Log4JConfigurationFactory() {
+    if (Log4JConfigurationFactory.logSettings == null) {
+      Log4JConfigurationFactory.logSettings = new MServerConfigManager(MServerConfigManager.DEFAULT_CONFIG_FILE).getConfig().getLogSettings();
+    }
+  }
+
   static Configuration createConfiguration(
       final String name, final ConfigurationBuilder<BuiltConfiguration> aBuilder) {
-    logSettings = new MServerConfigManager().getConfig().getLogSettings();
 
     aBuilder.setConfigurationName(name);
 

src/main/java/de/mediathekview/mserver/base/config/MServerConfigDTO.java

@@ -15,7 +15,7 @@ public class MServerConfigDTO extends MServerBasicConfigDTO implements ConfigDTO
   private final Boolean writeFilmlistIdFileEnabled;
   private final String filmlistIdFilePath;
   /** ignore certain film by title **/
-  private final String ignoreFilmlistPath;
+  private String ignoreFilmlistPath;
   /** add livestreams from external list **/
   private final ImportLivestreamConfiguration importLivestreamConfiguration;
   /** add additional filmlist from external **/
@@ -147,6 +147,10 @@ public Map<FilmlistFormats, String> getFilmlistSavePaths() {
   public void setFilmlistSavePaths(final Map<FilmlistFormats, String> filmlistSavePaths) {
     this.filmlistSavePaths = filmlistSavePaths;
   }
+
+  public void setIgnoreFilmlistPath(final String ignoreFilmlistPath) {
+    this.ignoreFilmlistPath = ignoreFilmlistPath;
+  }
 
   public MServerLogSettingsDTO getLogSettings() {
     return logSettings;

src/main/java/de/mediathekview/mserver/base/config/MServerConfigManager.java

@@ -14,10 +14,6 @@ public MServerConfigManager(final String fileName) {
     configFileName = fileName;
   }
 
-  public MServerConfigManager() {
-    this(DEFAULT_CONFIG_FILE);
-  }
-
   /**
    * @param aSender The {@link Sender} for which the config will be loaded.
    * @return The Sender specific config.

src/main/java/de/mediathekview/mserver/crawler/CrawlerManager.java

@@ -57,8 +57,8 @@ public class CrawlerManager extends AbstractManager {
   private static final String FILMLIST_JSON_COMPRESSED_DEFAULT_NAME =
       FILMLIST_JSON_DEFAULT_NAME + ".xz";
   private static final Logger LOG = LogManager.getLogger(CrawlerManager.class);
-  private static CrawlerManager instance;
   private final MServerConfigDTO config;
+  private final MServerConfigManager rootConfig;
   private final ForkJoinPool forkJoinPool;
   private final Filmlist filmlist;
   private final IgnoreFilmFilter ingoreFilmFilter;
@@ -70,9 +70,9 @@ public class CrawlerManager extends AbstractManager {
   private final Collection<ProgressListener> copyProgressListeners;
   private final Filmlist differenceList;
 
-  private CrawlerManager() {
+  public CrawlerManager(MServerConfigManager aMServerConfigManager) {
     super();
-    final MServerConfigManager rootConfig = new MServerConfigManager();
+    rootConfig = aMServerConfigManager;
     config = rootConfig.getConfig();
     ingoreFilmFilter = new IgnoreFilmFilter(config.getIgnoreFilmslistPath());
     executorService = Executors.newFixedThreadPool(config.getMaximumCpuThreads());
@@ -86,11 +86,8 @@ private CrawlerManager() {
     initializeCrawler(rootConfig);
   }
 
-  public static CrawlerManager getInstance() {
-    if (instance == null) {
-      instance = new CrawlerManager();
-    }
-    return instance;
+  public MServerConfigManager getConfigManager() {
+    return rootConfig;
   }
 
   public void copyFilmlist() {
@@ -166,6 +163,7 @@ public void importLivestreamFilmlist() {
   }
 
   public void importLivestreamFilmlist(final FilmlistFormats aFormat, final String aFilmlistLocation) {
+    LOG.debug("importLivestreamFilmlist {}", aFilmlistLocation);
     try {
       final Optional<Filmlist> importedFilmlist;
       if (aFilmlistLocation.startsWith(HTTP)) {

src/main/java/de/mediathekview/mserver/crawler/basic/IgnoreFilmFilter.java

@@ -31,7 +31,7 @@ public IgnoreFilmFilter(String configFileNameAndPath) {
       } else {
         ignoreFilmTitles = read(configFileNameAndPath);
       }
-      LOG.debug("ignoreFilmList setup with {} entries", size());
+      LOG.debug("ignoreFilmList {} setup with {} entries", configFileNameAndPath, size());
     } catch (IOException e) {
       LOG.error("Could not read ignorefilmlist from {} ",configFileNameAndPath, e);
     }

src/main/java/de/mediathekview/mserver/crawler/dw/DWTaskBase.java

@@ -5,7 +5,6 @@
 import com.google.gson.GsonBuilder;
 
 import de.mediathekview.mlib.daten.Sender;
-import de.mediathekview.mserver.base.config.MServerConfigManager;
 import de.mediathekview.mserver.crawler.basic.AbstractCrawler;
 import de.mediathekview.mserver.crawler.basic.AbstractRestTask;
 import de.mediathekview.mserver.crawler.basic.CrawlerUrlDTO;
@@ -25,9 +24,7 @@
 public abstract class DWTaskBase<T, D extends CrawlerUrlDTO> extends AbstractRestTask<T, D> {
   private static final Logger LOG = LogManager.getLogger(DWTaskBase.class);
 
-  private static final RateLimiter limiter =
-      RateLimiter.create(
-          new MServerConfigManager().getSenderConfig(Sender.DW).getMaximumRequestsPerSecond());
+  private static RateLimiter limiter = null;
 
   private final transient GsonBuilder gsonBuilder;
 
@@ -81,6 +78,9 @@ private Response executeRequest(final WebTarget aTarget) {
           request.header(
               ZdfConstants.HEADER_AUTHENTIFICATION, AUTHORIZATION_BEARER + authKey.get());
     }
+    if (limiter == null) {
+      limiter = RateLimiter.create(crawler.getRuntimeConfig().getSenderConfig(Sender.DW).getMaximumRequestsPerSecond());
+    }
     limiter.acquire();
     return request.header(HEADER_ACCEPT_ENCODING, ENCODING_GZIP).get();
   }

src/main/java/de/mediathekview/mserver/crawler/funk/json/FunkVideoDeserializer.java

@@ -73,9 +73,7 @@ protected FilmInfoDto mapToElement(final JsonObject jsonObject) {
   }
 
   private MServerConfigDTO getRuntimeConfig() {
-    return crawler
-        .map(AbstractCrawler::getRuntimeConfig)
-        .orElseGet(() -> new MServerConfigManager().getConfig());
+    return crawler.get().getRuntimeConfig();
   }
 
   private String createNexxCloudUrl(final String entityId) {

src/main/java/de/mediathekview/mserver/crawler/sr/tasks/SrFilmDetailTask.java

@@ -18,7 +18,6 @@
 import org.jsoup.nodes.Node;
 import org.jsoup.select.Elements;
 
-import java.io.IOException;
 import java.io.InputStreamReader;
 import java.net.MalformedURLException;
 import java.net.URL;

src/main/java/de/mediathekview/mserver/crawler/sr/tasks/SrRateLimitedDocumentTask.java

@@ -14,9 +14,7 @@ public abstract class SrRateLimitedDocumentTask<T, D extends CrawlerUrlDTO>
 
   private static final long serialVersionUID = -4077182368484515410L;
 
-  private static final RateLimiter LIMITER =
-      RateLimiter.create(
-          new MServerConfigManager().getSenderConfig(Sender.SR).getMaximumRequestsPerSecond());
+  private static RateLimiter LIMITER = null;
 
   SrRateLimitedDocumentTask(
       final AbstractCrawler crawler,
@@ -26,6 +24,9 @@ public abstract class SrRateLimitedDocumentTask<T, D extends CrawlerUrlDTO>
 
   @Override
   protected void processElement(final D urlDTO) {
+    if (LIMITER== null) {
+      LIMITER = RateLimiter.create(crawler.getRuntimeConfig().getSenderConfig(Sender.SR).getMaximumRequestsPerSecond());
+    }
     LIMITER.acquire();
     super.processElement(urlDTO);
   }

src/main/java/de/mediathekview/mserver/crawler/zdf/tasks/ZdfTaskBase.java

@@ -4,7 +4,6 @@
 import com.google.gson.Gson;
 import com.google.gson.GsonBuilder;
 import de.mediathekview.mlib.daten.Sender;
-import de.mediathekview.mserver.base.config.MServerConfigManager;
 import de.mediathekview.mserver.crawler.basic.AbstractCrawler;
 import de.mediathekview.mserver.crawler.basic.AbstractRestTask;
 import de.mediathekview.mserver.crawler.basic.CrawlerUrlDTO;
@@ -22,9 +21,7 @@
 public abstract class ZdfTaskBase<T, D extends CrawlerUrlDTO> extends AbstractRestTask<T, D> {
   private static final Logger LOG = LogManager.getLogger(ZdfTaskBase.class);
 
-  private static final RateLimiter limiter =
-      RateLimiter.create(
-          new MServerConfigManager().getSenderConfig(Sender.ZDF).getMaximumRequestsPerSecond());
+  private static RateLimiter limiter = null;
 
   private final GsonBuilder gsonBuilder;
 
@@ -76,6 +73,10 @@ private Response executeRequest(final WebTarget aTarget) {
           request.header(
               ZdfConstants.HEADER_AUTHENTIFICATION, AUTHORIZATION_BEARER + authKey.get());
     }
+    if (limiter == null) {
+      limiter = RateLimiter.create(crawler.getRuntimeConfig().getSenderConfig(Sender.ZDF).getMaximumRequestsPerSecond());
+    }
+
     limiter.acquire();
     return request.header(HEADER_ACCEPT_ENCODING, ENCODING_GZIP).get();
   }

src/main/java/de/mediathekview/mserver/ui/config/MServerConfigUI.java

@@ -2,7 +2,7 @@
 
 import de.mediathekview.mlib.messages.listener.LogMessageListener;
 import de.mediathekview.mlib.messages.listener.MessageListener;
-import de.mediathekview.mserver.base.config.MServerConfigDTO;
+import de.mediathekview.mserver.base.config.Log4JConfigurationFactory;
 import de.mediathekview.mserver.base.config.MServerConfigManager;
 import de.mediathekview.mserver.base.config.MServerLogSettingsDTO;
 import de.mediathekview.mserver.base.messages.ServerMessages;
@@ -13,34 +13,27 @@
 import org.apache.logging.log4j.Logger;
 
 import java.io.IOException;
+import java.io.InputStream;
+import java.net.MalformedURLException;
 import java.net.URISyntaxException;
+import java.net.URL;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
+import java.nio.file.StandardCopyOption;
 import java.util.ArrayList;
 import java.util.List;
 
 public final class MServerConfigUI {
-
-  private static final Logger LOG = LogManager.getLogger(MServerConfigUI.class);
+  // logger setup in start
+  private Logger LOG = null;
   private static final String CONFIG_FILE_NAME = "MServer-Config.yaml";
   private static final String ARGUMENT_GCONF = "-gconf";
-  private final LogMessageListener logMessageListener;
-  private final MServerConfigDTO config;
+  private LogMessageListener logMessageListener;
   private CrawlerManager manager;
 
   public MServerConfigUI() {
     super();
-    config = new MServerConfigManager().getConfig();
-    final MServerLogSettingsDTO logSettings = config.getLogSettings();
-    logSettings.setLogActivateConsole(true);
-
-    logMessageListener = new LogMessageListener();
-
-    final Level configLevel = logSettings.getLogLevelConsole();
-    if (configLevel == null || !logLevelInfoOrLower(configLevel)) {
-      logSettings.setLogLevelConsole(Level.INFO);
-    }
   }
 
   public static void main(final String[] args) {
@@ -79,10 +72,6 @@ private boolean interpretProgramArguments(final String[] aProgramAgruments) {
 
       if (ARGUMENT_GCONF.equals(aProgramAgruments[0])) {
         generateDefaultConfiguration();
-      } else {
-        logMessageListener.consumeMessage(
-            ServerMessages.UI_UNKNOWN_ARGUMENT, aProgramAgruments[0], ARGUMENT_GCONF);
-        return false;
       }
     }
 
@@ -99,9 +88,7 @@ void start() {
     try {
       manager.start();
       manager.importFilmlist();
-      if (Boolean.TRUE.equals(config.getImportLivestreamConfiguration().isActive())) {
-        manager.importLivestreamFilmlist();
-      }
+      manager.importLivestreamFilmlist();
     } finally {
       manager.filterFilmlist();
       manager.saveFilmlist();
@@ -114,10 +101,53 @@ void start() {
   }
 
   void start(final String[] aProgramAgruments) {
+    MServerConfigManager aMServerConfigManager = null;
+    if (aProgramAgruments.length > 0 && !ARGUMENT_GCONF.equals(aProgramAgruments[0])) {
+      String configFileName = aProgramAgruments[0];
+      if (configFileName.startsWith("http")) {
+        URL fileUrl;
+        try {
+          // get a copy of this file to use it as configuration file
+          fileUrl = new URL(configFileName);
+          String filename = Paths.get(fileUrl.getPath()).getFileName().toString();
+          MServerConfigUI.getRemoteFileToLocal(configFileName, filename);
+          configFileName = filename;
+        } catch (MalformedURLException e) {
+          e.printStackTrace();
+        }
+      }
+      aMServerConfigManager = new MServerConfigManager(configFileName);
+    } else {
+      aMServerConfigManager = new MServerConfigManager(MServerConfigManager.DEFAULT_CONFIG_FILE);
+    }
+    // here we set the correct configManager for all log4logger
+    // logsettings are stored static in our factory
+    new Log4JConfigurationFactory(aMServerConfigManager.getConfig().getLogSettings());
+    LOG = LogManager.getLogger(MServerConfigUI.class);
+    logMessageListener = new LogMessageListener();
+
     if (interpretProgramArguments(aProgramAgruments)) {
-      manager = CrawlerManager.getInstance();
+      manager = new CrawlerManager(aMServerConfigManager);
+      final MServerLogSettingsDTO logSettings = aMServerConfigManager.getConfig().getLogSettings();
+      logSettings.setLogActivateConsole(true);
+      final Level configLevel = logSettings.getLogLevelConsole();
+      if (configLevel == null || !logLevelInfoOrLower(configLevel)) {
+        logSettings.setLogLevelConsole(Level.INFO);
+      }
       addListeners();
       start();
     }
   }
+
+  public static void getRemoteFileToLocal(String source, String target) {
+    try {
+        URL fileUrl = new URL(source);
+        try (InputStream in = fileUrl.openStream()) {
+            Path outputPath = Path.of(target);
+            Files.copy(in, outputPath, StandardCopyOption.REPLACE_EXISTING);
+        }
+    } catch (IOException e) {
+        e.printStackTrace(); // we do not have a logger yet
+    }
+  }
 }

src/test/java/de/mediathekview/mserver/crawler/CrawlerManagerImportFilmlistsTest.java

@@ -6,6 +6,7 @@
 import de.mediathekview.mlib.messages.MessageUtil;
 import de.mediathekview.mlib.messages.listener.MessageListener;
 import de.mediathekview.mserver.base.config.ImportFilmlistConfiguration;
+import de.mediathekview.mserver.base.config.MServerConfigManager;
 import de.mediathekview.mserver.testhelper.FileReader;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
@@ -24,7 +25,6 @@
 
 import java.io.File;
 import java.io.IOException;
-import java.lang.reflect.Field;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.Comparator;
@@ -34,7 +34,7 @@
 
 public class CrawlerManagerImportFilmlistsTest implements MessageListener {
 
-  private static final Logger LOG = LogManager.getLogger(CrawlerManagerImportFilmlistsTest.class);
+  private Logger LOG;
   private static final String TEMP_FOLDER_NAME_PATTERN = "MSERVER_TEST_%d";
   private static Path testFileFolderPath;
 
@@ -87,16 +87,9 @@ void testSaveAndImport(final ImportFilmlistConfiguration initialList, final Impo
   }
 
   public CrawlerManager createEmptyCrawlerManager() {
-    // reset singelton CrawlerManager to have an empty filmlist
-    Field instance;
-    try {
-      instance = CrawlerManager.class.getDeclaredField("instance");
-      instance.setAccessible(true);
-      instance.set(null, null);
-    } catch (Exception e) {
-      fail("Exception mooking crawler manager: " + e.getMessage());
-    }    //
-    return CrawlerManager.getInstance();
+    CrawlerManager cm = new CrawlerManager(new MServerConfigManager(MServerConfigManager.DEFAULT_CONFIG_FILE));
+    LOG = LogManager.getLogger(CrawlerManagerImportFilmlistsTest.class);
+    return cm;
   }
 
   @AfterAll