Skip to content

Commit

Permalink
#156 - check point
Browse files Browse the repository at this point in the history
  • Loading branch information
hohonuuli committed Apr 13, 2023
1 parent a9b8421 commit ea9cc1f
Show file tree
Hide file tree
Showing 11 changed files with 237 additions and 40 deletions.
2 changes: 1 addition & 1 deletion docs/setup.md
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,7 @@ Then on the network tab set the control port:

#### ML Configuration

VARS can send the current video frame to a remote server where machine learning can be applied to the image. To configure, simply enter the URL to the ML endpoint. For internal use, that endpoint is: <http://digits-dev-box-fish.shore.mbari.org:8082/predictor/>
VARS can send the current video frame to a remote server where machine learning can be applied to the image. To configure, simply enter the URL to the ML endpoint. For internal use, that endpoint is: <http://perceptron.shore.mbari.org:8080/predictor/>

![Machine Learning Endpoint](assets/images/MachineLearningConfiguration.png)

Expand Down
Original file line number Diff line number Diff line change
@@ -1,10 +1,15 @@
package org.mbari.vars.ui.commands;

import org.mbari.vars.services.model.BoundingBox;
import org.mbari.vars.ui.UIToolBox;
import org.mbari.vars.services.model.Annotation;
import org.mbari.vars.services.model.Association;
import org.mbari.vars.services.AnnotationService;
import org.mbari.vars.ui.javafx.AnnotationServiceDecorator;
import org.mbari.vars.ui.mediaplayers.sharktopoda2.LocalizedAnnotation;
import org.mbari.vcr4j.remote.control.RVideoIO;
import org.mbari.vcr4j.remote.control.commands.localization.AddLocalizationsCmd;
import org.mbari.vcr4j.remote.control.commands.localization.RemoveLocalizationsCmd;

import java.util.*;
import java.util.concurrent.CompletableFuture;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -134,14 +134,17 @@ public FlowPane getButtonPane() {
Button mlBtn = new JFXButton();
new MachineLearningBC(mlBtn, toolBox);

Button bbBtn = new JFXButton();
new NewBoundingBoxBC(bbBtn, toolBox);

// Button denseBtn = new JFXButton();
// new TempDenseBC(denseBtn, toolBox);
//
// Button ninesBtn = new JFXButton();
// new TempPopulationNinesBC(ninesBtn, toolBox);

buttonPane.getChildren().addAll(newBtn, dupBtn, copyBtn, framegrabBtn,
detachBtn, mlBtn, sampleBtn, newRefBtn, oldRefBtn, uponBtn, pqBtn,
detachBtn, mlBtn, bbBtn, sampleBtn, newRefBtn, oldRefBtn, uponBtn, pqBtn,
commentBtn, durationBtn, deleteDurationBtn, deleteBtn);

}
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,80 @@
package org.mbari.vars.ui.javafx.buttons;

import javafx.collections.ListChangeListener;
import javafx.scene.control.Button;
import javafx.scene.text.Text;
import org.mbari.vars.services.model.Annotation;
import org.mbari.vars.services.model.Association;
import org.mbari.vars.services.model.Media;
import org.mbari.vars.ui.UIToolBox;
import org.mbari.vars.ui.commands.CreateAssociationsCmd;
import org.mbari.vars.ui.javafx.Icons;
import org.mbari.vars.ui.mediaplayers.sharktopoda2.LocalizedAnnotation;
import org.mbari.vcr4j.remote.control.commands.localization.Localization;

import java.util.ArrayList;
import java.util.List;
import java.util.UUID;

public class NewBoundingBoxBC extends AbstractBC {

public NewBoundingBoxBC(Button button, UIToolBox toolBox) {
super(button, toolBox);
}

@Override
protected void apply() {
var selectedAnnotations = new ArrayList<>(toolBox.getData().getSelectedAnnotations());
if (selectedAnnotations.size() == 1) {
var annotation = selectedAnnotations.get(0);
var media = toolBox.getData().getMedia();
if (canLocalize(media, annotation)) {
var association = toAssociation(annotation, media);
association.resetUuid();
toolBox.getEventBus().send(new CreateAssociationsCmd(association, List.of(annotation)));
}
}
}

private boolean canLocalize(Media media, Annotation annotation) {
return media != null
&& media.getWidth() != null
&& media.getHeight() != null
&& annotation.getElapsedTime() != null
&& annotation.getVideoReferenceUuid().equals(media.getVideoReferenceUuid());
}

private Association toAssociation(Annotation annotation, Media media) {
var w = media.getWidth();
var h = media.getHeight();
int width = (int) Math.round(w * 0.1);
int height = (int) Math.round(h * 0.1);
int x = (int) Math.round((w / 2.0) - (width / 2.0));
int y = (int) Math.round((h / 2.0) - (height / 2.0));
var duration = annotation.getDuration() == null ? null : annotation.getDuration().toMillis();
var localization = new Localization(UUID.randomUUID(),
annotation.getConcept(),
annotation.getElapsedTime().toMillis(),
duration,
x, y, width, height, null);
return LocalizedAnnotation.toAssociation(localization);
}

@Override
protected void init() {
String tooltip = toolBox.getI18nBundle().getString("buttons.boundingbox");
Text icon = Icons.FORMAT_SHAPES.standardSize();
initializeButton(tooltip, icon);
var selectedAnnotations = toolBox.getData().getSelectedAnnotations();
selectedAnnotations
.addListener((ListChangeListener<Annotation>) c -> {
var disable = true;
if (selectedAnnotations.size() == 1) {
var head = selectedAnnotations.get(0);
var media = toolBox.getData().getMedia();
disable = !canLocalize(media, head);
}
button.setDisable(disable);
});
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,8 @@

public record LocalizedAnnotation(Annotation annotation, Association association) {

public static String GENERATOR = "VARS Annotation";

private static Gson gson = new GsonBuilder()
.setDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'")
.registerTypeAdapter(Duration.class, new DurationConverter())
Expand Down Expand Up @@ -63,7 +65,7 @@ public static LocalizedAnnotation from(Localization x) {
}

public static Association toAssociation(Localization x) {
BoundingBox bb = new BoundingBox(x.getX(), x.getY(), x.getWidth(), x.getHeight(), "VARS Annotation");
BoundingBox bb = new BoundingBox(x.getX(), x.getY(), x.getWidth(), x.getHeight(), GENERATOR);
String json = gson.toJson(bb);

return new Association(BoundingBox.LINK_NAME,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -117,18 +117,19 @@ private void handleSelect(List<UUID> selected) {
log.info("An attempt was made to update a localization, but no media is currently open");
return;
}
var matches = searchByUuid(selected);
var matchingLocalizationUuids = matches.stream()
.map(lp -> lp.localization().getUuid())
.toList();
sharktopodaState.setSelectedLocalizations(matchingLocalizationUuids);
// Only send this if the selected are different from the currently selected
if (sharktopodaState.isDifferentThanSelected(selected)) {
var matches = searchByUuid(selected);
sharktopodaState.setSelectedLocalizations(matches);

var selectedAnnotations = matches.stream()
.map(LocalizationPair::localizedAnnotation)
.map(LocalizedAnnotation::annotation)
.toList();
var cmd = new AnnotationsSelectedEvent(this, selectedAnnotations);
toolBox.getEventBus().send(cmd);
var selectedAnnotations = matches.stream()
.map(LocalizationPair::localizedAnnotation)
.map(LocalizedAnnotation::annotation)
.toList();

var cmd = new AnnotationsSelectedEvent(this, selectedAnnotations);
toolBox.getEventBus().send(cmd);
}
}

private List<LocalizationPair> searchByUuid(List<UUID> uuids) {
Expand Down
Original file line number Diff line number Diff line change
@@ -1,13 +1,37 @@
package org.mbari.vars.ui.mediaplayers.sharktopoda2;

import org.mbari.vars.services.model.Annotation;
import org.mbari.vars.ui.UIToolBox;
import org.mbari.vcr4j.remote.control.commands.localization.Localization;

import java.util.List;

/**
* When the remote video player sends a command to update, select, or delete a localization to VARS,
* the localization's UUID corresponds to an existing Association. We then have to match the
* localization to the assocation and it's parent annotations. This container holds all three of those.
* @param localizedAnnotation The Annotation and association corresponding to a localization
* @param localization The localization send from the video app to VARS
*/
public record LocalizationPair(LocalizedAnnotation localizedAnnotation, Localization localization) {
public record LocalizationPair(LocalizedAnnotation localizedAnnotation,
Localization localization) implements Comparable<LocalizationPair> {

@Override
public boolean equals(Object obj) {
if (obj instanceof LocalizationPair that) {
return this.localization.getUuid().equals(that.localization.getUuid());
}
return false;
}

@Override
public int hashCode() {
return localization.getUuid().hashCode();
}

@Override
public int compareTo(LocalizationPair that) {
return this.localization.getUuid().compareTo(that.localization.getUuid());
}

}
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,6 @@
import org.mbari.vars.services.model.Annotation;
import org.mbari.vars.services.model.Association;
import org.mbari.vars.services.model.BoundingBox;
import org.mbari.vars.services.model.Media;
import org.mbari.vars.ui.AppConfig;
import org.mbari.vars.ui.UIToolBox;
import org.mbari.vcr4j.remote.control.commands.localization.Localization;
import org.mbari.vcr4j.sharktopoda.client.gson.DurationConverter;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -64,11 +64,12 @@ private void init(EventBus eventBus) {
}

private void handle(Collection<Annotation> annotations, Action action) {
var media = toolBox.getData().getMedia();
List<Localization> localizations = LocalizedAnnotation.from(annotations)
.stream()
.flatMap(opt -> opt.toLocalization(toolBox).stream())
.toList();

List<LocalizationPair> localizationPairs = SharktopodaState.from(annotations, toolBox);

List<Localization> localizations = localizationPairs.stream()
.map(LocalizationPair::localization)
.toList();

log.atDebug().log(() -> "Outgoing to Sharktopoda: %s on %d localizations".formatted(action, localizations.size()));

Expand All @@ -77,22 +78,35 @@ private void handle(Collection<Annotation> annotations, Action action) {
.map(Localization::getUuid)
.toList();
switch (action) {
case Add -> io.send(new AddLocalizationsCmd(io.getUuid(), localizations));
case Update -> io.send(new UpdateLocalizationsCmd(io.getUuid(), localizations));
case Remove -> io.send(new RemoveLocalizationsCmd(io.getUuid(), uuids));
case Add -> {
sharktopodaState.addLocalizationPairs(localizationPairs);
io.send(new AddLocalizationsCmd(io.getUuid(), localizations));
}
case Update -> {
// TODO For Changed events we need to remove any localizations that
// on the annotation that no longer exist

// TODO for changed events we may have to add a new localization that
// was created on the VARS side
sharktopodaState.addLocalizationPairs(localizationPairs);
io.send(new UpdateLocalizationsCmd(io.getUuid(), localizations));
}
case Remove -> {
sharktopodaState.removeLocalizationPairs(localizationPairs);
io.send(new RemoveLocalizationsCmd(io.getUuid(), uuids));
}
case Select -> {
if (sharktopodaState.isDifferentThanSelected(uuids)) {
sharktopodaState.setSelectedLocalizations(uuids);
sharktopodaState.setSelectedLocalizations(localizationPairs);
io.send(new SelectLocalizationsCmd(io.getUuid(), uuids));
}
}
}
}

if (action.equals(Action.Select) && localizations.isEmpty() && !annotations.isEmpty()) {
List<UUID> nothing = Collections.emptyList();
sharktopodaState.setSelectedLocalizations(nothing);
io.send(new SelectLocalizationsCmd(io.getUuid(), nothing));
sharktopodaState.setSelectedLocalizations(Collections.emptyList());
io.send(new SelectLocalizationsCmd(io.getUuid(), Collections.emptyList()));
}
}

Expand Down
Original file line number Diff line number Diff line change
@@ -1,34 +1,103 @@
package org.mbari.vars.ui.mediaplayers.sharktopoda2;

import org.mbari.vars.services.model.Annotation;
import org.mbari.vars.ui.UIToolBox;
import org.mbari.vcr4j.remote.control.RVideoIO;
import org.mbari.vcr4j.sharktopoda.client.localization.Localization;

import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import java.util.UUID;
import java.util.*;
import java.util.concurrent.CopyOnWriteArrayList;

public class SharktopodaState {

private final List<UUID> selectedLocalizations = new CopyOnWriteArrayList<>();
// private final List<UUID> selectedLocalizations = new CopyOnWriteArrayList<>();

public void setSelectedLocalizations(Collection<UUID> selectedLocalizations) {
private final List<LocalizationPair> selectedLocalizations = new CopyOnWriteArrayList<>();

private final Map<UUID, LocalizationPair> localizations = new HashMap<>();

public void setSelectedLocalizations(Collection<LocalizationPair> selectedLocalizations) {
synchronized (this.selectedLocalizations) {
this.selectedLocalizations.clear();
this.selectedLocalizations.addAll(selectedLocalizations);
}
}

public void clear() {
selectedLocalizations.clear();
localizations.clear();
}

public void addLocalizationPairs(Collection<LocalizationPair> pairs) {
pairs.forEach(this::addLocalizationPair);
}

public void addLocalizationPair(LocalizationPair localizationPair) {
localizations.put(localizationPair.localization().getUuid(), localizationPair);
}

public void removeLocalizationPairs(Collection<LocalizationPair> pairs) {
pairs.forEach(p -> removeLocalizationPair(p.localization().getUuid()));
}

public void removeLocalizationPair(UUID localizationUuid) {
localizations.remove(localizationUuid);
}


public boolean isDifferentThanSelected(Collection<UUID> localizations) {
var isDiff = true;
if (!localizations.isEmpty() && !selectedLocalizations.isEmpty()) {
if (localizations.isEmpty() && selectedLocalizations.isEmpty()) {
return false;
}
else if (!localizations.isEmpty() && !selectedLocalizations.isEmpty()) {
var copy = new ArrayList<>(localizations);
var copySelected = new ArrayList<>(selectedLocalizations);
var copySelected = selectedLocalizations.stream().map(lp -> lp.localization().getUuid()).toList();
if (copy.size() == copySelected.size()) {
copy.removeAll(copySelected);
isDiff = copy.size() != 0;
return copy.size() != 0;
}
}
return isDiff;
return true;
}

public static List<LocalizationPair> from(Annotation a, UIToolBox toolBox) {
return LocalizedAnnotation.from(a)
.stream()
.flatMap(la -> la.toLocalization(toolBox).map(loc -> new LocalizationPair(la, loc)).stream())
.toList();
}

public static List<LocalizationPair> from(Collection<Annotation> xs, UIToolBox toolBox) {
return xs.stream()
.flatMap(a -> SharktopodaState.from(a, toolBox).stream())
.toList();
}

// TODO MOve this to static shared method for reuse
// public static void addLocalization(Collection<Annotation> annotations, UIToolBox toolBox) {
// var head = annotations.iterator().next();
// var ass = addedAssociations.iterator().next();
// var io = toolBox.getMediaPlayer().getVideoIO();
// if (io instanceof RVideoIO rio) {
// var la = new LocalizedAnnotation(head, ass);
// var opt = la.toLocalization(toolBox);
// opt.ifPresent(loc -> {
// rio.send(new AddLocalizationsCmd(rio.getUuid(), List.of(loc)));
// });
// }
// }
//
// // TODO MOve this to static shared method for reuse
// public static void removeLocalization(Collection<Annotation> originalAnnotations, UIToolBox toolBox) {
// var head = originalAnnotations.iterator().next();
// var ass = addedAssociations.iterator().next();
// var io = toolBox.getMediaPlayer().getVideoIO();
// if (io instanceof RVideoIO rio) {
// var la = new LocalizedAnnotation(head, ass);
// var opt = la.toLocalization(toolBox);
// opt.ifPresent(loc -> {
// rio.send(new RemoveLocalizationsCmd(rio.getUuid(), List.of(loc.getUuid())));
// });
// }
// }
}
Loading

0 comments on commit ea9cc1f

Please sign in to comment.