Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Hotfix/mardv6 #984

Merged
merged 7 commits into from
May 17, 2024
Merged
Show file tree
Hide file tree
Changes from 4 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 8 additions & 4 deletions src/main/java/mServer/crawler/FilmeSuchen.java
Original file line number Diff line number Diff line change
Expand Up @@ -139,11 +139,15 @@ public synchronized void filmeBeimSenderLaden(ListeFilme listeFilme) {
initStart(listeFilme);
// die mReader nach Prio starten
mrStarten(0);
if (!Config.getStop()) {
// waren und wenn Suchlauf noch nicht abgebrochen weiter mit dem Rest
mrWarten(4*60);//4*60);
mrStarten(1);
if (mediathekListe.stream().filter(mr -> mr.getStartPrio() == 1).count() == 0) {
allStarted = true;
} else {
if (!Config.getStop()) {
// waren und wenn Suchlauf noch nicht abgebrochen weiter mit dem Rest
mrWarten(4*60);//4*60);
mrStarten(1);
allStarted = true;
}
}
}

Expand Down
4 changes: 3 additions & 1 deletion src/main/java/mServer/crawler/sender/ard/ArdConstants.java
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ public class ArdConstants {
public static final String API_URL = "https://api.ardmediathek.de";
public static final String BASE_URL = "https://api.ardmediathek.de/public-gateway";

public static final String ITEM_URL = API_URL + "/page-gateway/pages/ard/item/";
public static final String ITEM_URL = API_URL + "/page-gateway/pages/ard/item/%s?embedded=true&mcV6=true";

public static final String TOPICS_URL = API_URL + "/page-gateway/pages/%s/editorial/experiment-a-z?embedded=false";
public static final String TOPICS_COMPILATION_URL = API_URL + "/page-gateway/widgets/%s/editorials/%s?pageNumber=0&pageSize=%s";
Expand Down Expand Up @@ -37,6 +37,8 @@ public class ArdConstants {
"phoenix"
};

public static final String[] IGNORED_SENDER = new String[] {"zdf", "kika", "3sat", "arte"};

public static final String WEBSITE_URL = "https://www.ardmediathek.de/video/%s";

public static final String BASE_URL_SUBTITLES = "https://classic.ardmediathek.de";
Expand Down
5 changes: 5 additions & 0 deletions src/main/java/mServer/crawler/sender/ard/ArdCrawler.java
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,11 @@ protected synchronized void meldungThreadUndFertig() {
mlibFilmeSuchen.meldenFertig(Const.HR);
mlibFilmeSuchen.meldenFertig(Const.BR);
mlibFilmeSuchen.meldenFertig("rbtv");
mlibFilmeSuchen.meldenFertig("ONE");
mlibFilmeSuchen.meldenFertig("ARD-alpha");
mlibFilmeSuchen.meldenFertig("Funk.net");
mlibFilmeSuchen.meldenFertig(Const.SR);
codingPF marked this conversation as resolved.
Show resolved Hide resolved
mlibFilmeSuchen.meldenFertig(Const.PHOENIX);
}

super.meldungThreadUndFertig();
Expand Down
Original file line number Diff line number Diff line change
@@ -1,35 +1,34 @@
package mServer.crawler.sender.ard.json;

import com.google.gson.JsonArray;
import com.google.gson.JsonDeserializationContext;
import com.google.gson.JsonDeserializer;
import com.google.gson.JsonElement;
import com.google.gson.JsonObject;
import com.google.gson.*;

import mServer.crawler.sender.ard.ArdFilmInfoDto;

import java.lang.reflect.Type;
import java.util.HashSet;
import java.util.Set;
import mServer.crawler.sender.ard.ArdFilmInfoDto;

public class ArdDayPageDeserializer extends ArdTeasersDeserializer implements JsonDeserializer<Set<ArdFilmInfoDto>> {
public class ArdDayPageDeserializer extends ArdTeasersDeserializer
implements JsonDeserializer<Set<ArdFilmInfoDto>> {

private static final String ELEMENT_TEASERS = "teasers";

@Override
public Set<ArdFilmInfoDto> deserialize(JsonElement jsonElement, Type type, JsonDeserializationContext context) {
Set<ArdFilmInfoDto> results = new HashSet<>();
public Set<ArdFilmInfoDto> deserialize(
final JsonElement jsonElement, final Type type, final JsonDeserializationContext context) {
final Set<ArdFilmInfoDto> results = new HashSet<>();

if (!jsonElement.isJsonArray()) {
return results;
}

JsonObject element0 = jsonElement.getAsJsonArray().get(0).getAsJsonObject();
final JsonObject firstElement = jsonElement.getAsJsonArray().get(0).getAsJsonObject();

if (element0.has(ELEMENT_TEASERS)) {
JsonArray teasers = element0.get(ELEMENT_TEASERS).getAsJsonArray();
if (firstElement.has(ELEMENT_TEASERS)) {
final JsonArray teasers = firstElement.get(ELEMENT_TEASERS).getAsJsonArray();
results.addAll(parseTeasers(teasers));
}

return results;
}

}
Original file line number Diff line number Diff line change
@@ -1,13 +1,11 @@
package mServer.crawler.sender.ard.json;

import com.google.gson.JsonArray;
import com.google.gson.JsonDeserializationContext;
import com.google.gson.JsonDeserializer;
import com.google.gson.JsonElement;
import com.google.gson.JsonObject;
import com.google.gson.*;

import mServer.crawler.sender.base.JsonUtils;

import java.lang.reflect.Type;
import java.util.Optional;
import mServer.crawler.sender.base.JsonUtils;

public class ArdErrorDeserializer implements JsonDeserializer<Optional<ArdErrorInfoDto>> {

Expand All @@ -17,29 +15,32 @@ public class ArdErrorDeserializer implements JsonDeserializer<Optional<ArdErrorI
private static final String ELEMENT_EXTENSIONS = "extensions";

@Override
public Optional<ArdErrorInfoDto> deserialize(JsonElement jsonElement, Type type, JsonDeserializationContext context) {
public Optional<ArdErrorInfoDto> deserialize(
final JsonElement jsonElement, final Type type, final JsonDeserializationContext context) {

if (!JsonUtils.hasElements(jsonElement, ELEMENT_ERRORS)) {
return Optional.empty();
}

JsonArray errors = jsonElement.getAsJsonObject().get(ELEMENT_ERRORS).getAsJsonArray();
final JsonArray errors = jsonElement.getAsJsonObject().get(ELEMENT_ERRORS).getAsJsonArray();
if (errors.size() > 0) {
return parseError(errors.get(0).getAsJsonObject());
}

return Optional.empty();
}

private Optional<ArdErrorInfoDto> parseError(JsonObject error) {
Optional<String> message = JsonUtils.getAttributeAsString(error, ATTRIBUTE_MESSAGE);
private Optional<ArdErrorInfoDto> parseError(final JsonObject error) {
final Optional<String> message = JsonUtils.getAttributeAsString(error, ATTRIBUTE_MESSAGE);
Optional<String> code = Optional.empty();

if (JsonUtils.hasElements(error, ELEMENT_EXTENSIONS)) {
code = JsonUtils.getAttributeAsString(error.get(ELEMENT_EXTENSIONS).getAsJsonObject(), ATTRIBUTE_CODE);
code =
JsonUtils.getAttributeAsString(
error.get(ELEMENT_EXTENSIONS).getAsJsonObject(), ATTRIBUTE_CODE);
}

ArdErrorInfoDto result = new ArdErrorInfoDto(code.orElse(""), message.orElse(""));
final ArdErrorInfoDto result = new ArdErrorInfoDto(code.orElse(""), message.orElse(""));
return Optional.of(result);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -4,10 +4,10 @@

public class ArdErrorInfoDto {

private String code;
private String message;
private final String code;
private final String message;

public ArdErrorInfoDto(String code, String message) {
public ArdErrorInfoDto(final String code, final String message) {
this.code = code;
this.message = message;
}
Expand All @@ -21,14 +21,13 @@ public String getCode() {
}

@Override
public boolean equals(Object o) {
public boolean equals(final Object o) {
if (this == o) {
return true;
}
if (!(o instanceof ArdErrorInfoDto)) {
if (!(o instanceof ArdErrorInfoDto that)) {
return false;
}
ArdErrorInfoDto that = (ArdErrorInfoDto) o;
return Objects.equals(code, that.code) &&
Objects.equals(message, that.message);
}
Expand Down
Loading
Loading