Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

add tomestone flag bit into OptOutEntry to avoid major changes #57

Open
wants to merge 1 commit into
base: main
Choose a base branch
from
+84 −12
Open

add tomestone flag bit into OptOutEntry to avoid major changes #57

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
49 changes: 40 additions & 9 deletions src/main/java/com/uid2/shared/optout/OptOutEntry.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,8 @@
// Total: 72 bytes
//
// Metadata format:
// Lower 6 bits -- identity type
// Lower 5 bits -- identity type
// Middle 1 bit -- tombstone
// Higher 2 bits -- record version
// Total: 1 byte
//
Expand All @@ -23,18 +24,33 @@ public final class OptOutEntry {
public final byte[] identityHash;
public final byte[] advertisingId;
public final long timestamp;
public final boolean isTombstone;

private static final long timestampMask = 0xFFFFFFFFFFFFFFl;
private static final byte adsIdTypeMask = 0x3F;
private static final long timestampMask = 0xFFFFFFFFFFFFFFL;
private static final byte adsIdTypeMask = 0x1F;
private static final int adsIdVersionShift = 6;
private static final byte tombstoneMask = 0x1;
private static final int tombstoneShift = 5;

@Deprecated
public OptOutEntry(byte[] identityHash, byte[] advertisingId, long ts) {
assert identityHash.length == OptOutConst.Sha256Bytes;
assert advertisingId.length == OptOutConst.Sha256Bytes || advertisingId.length == OptOutConst.Sha256Bytes + 1;
// assert ts >= 0;
this.identityHash = identityHash;
this.advertisingId = advertisingId;
this.timestamp = ts;
this.isTombstone = false;
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can you call the constructor below instead?

}

public OptOutEntry(byte[] identityHash, byte[] advertisingId, long ts, boolean isTombstone) {
assert identityHash.length == OptOutConst.Sha256Bytes;
assert advertisingId.length == OptOutConst.Sha256Bytes || advertisingId.length == OptOutConst.Sha256Bytes + 1;
// assert ts >= 0;
this.identityHash = identityHash;
this.advertisingId = advertisingId;
this.timestamp = ts;
this.isTombstone = isTombstone;
}

public static OptOutEntry parse(byte[] buffer, int bufferIndex) {
Expand All @@ -43,6 +59,7 @@ public static OptOutEntry parse(byte[] buffer, int bufferIndex) {
final byte metadata = buffer[bufferIndex + OptOutConst.EntrySize - 1];
final byte adsIdVersion = (byte) (metadata >> adsIdVersionShift);
final byte identityType = (byte) (metadata & adsIdTypeMask);
final boolean isTombstone = (((byte) (metadata >> tombstoneShift)) & tombstoneMask) == 0x1;

final byte[] idHash = Arrays.copyOfRange(buffer, bufferIndex, bufferIndex + OptOutConst.Sha256Bytes);
bufferIndex += OptOutConst.Sha256Bytes;
Expand All @@ -54,7 +71,7 @@ public static OptOutEntry parse(byte[] buffer, int bufferIndex) {

final long ts = ByteBuffer.wrap(buffer, bufferIndex, Long.BYTES).order(ByteOrder.LITTLE_ENDIAN).getLong() & timestampMask;

return new OptOutEntry(idHash, adsId, ts);
return new OptOutEntry(idHash, adsId, ts, isTombstone);
}

private static byte[] parseAdsIdV3(byte[] buffer, int bufferIndex, byte identityType) {
Expand All @@ -70,6 +87,12 @@ public static long parseTimestamp(byte[] buffer, int bufferIndexForEntry) {
.order(ByteOrder.LITTLE_ENDIAN).getLong() & timestampMask;
}

public static boolean parseTombstone(byte[] buffer, int bufferIndexForEntry) {
assert bufferIndexForEntry + OptOutConst.EntrySize <= buffer.length;
byte metadata = buffer[bufferIndexForEntry + (OptOutConst.Sha256Bytes << 1) + 7];
return ((byte)(metadata >> tombstoneShift) & tombstoneMask) == (byte)0x1;
}

public static void setTimestamp(byte[] buffer, int bufferIndexForEntry, long timestamp) {
assert bufferIndexForEntry + OptOutConst.EntrySize <= buffer.length;
final byte metadata = buffer[bufferIndexForEntry + OptOutConst.EntrySize - 1];
Expand Down Expand Up @@ -109,7 +132,7 @@ public static OptOutEntry newRandom() {
public static OptOutEntry newTestEntry(long idHash, long timestamp) {
// for test, using the same value for identity_hash and advertising_id
byte[] id = idHashFromLong(idHash);
return new OptOutEntry(id, id, timestamp);
return new OptOutEntry(id, id, timestamp, false);
}

// Overriding equals() to compare two OptOutEntry objects
Expand All @@ -131,16 +154,19 @@ public int hashCode() {
return (int) (timestamp + Arrays.hashCode(identityHash) + Arrays.hashCode(advertisingId));
}

private static byte calcMetadata(byte[] advertisingId) {
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Do we need to leave the old function in for backward compatibility?

return (byte) (advertisingId.length == OptOutConst.Sha256Bytes ? 0 : ((1 << adsIdVersionShift) | advertisingId[0]));
private static byte calcMetadata(byte[] advertisingId, boolean isTombstone) {
return (byte) (
(advertisingId.length == OptOutConst.Sha256Bytes ? 0 : ((1 << adsIdVersionShift) | advertisingId[0]))
| (isTombstone ? (1 << tombstoneShift) : 0)
);
}

public void copyToByteArray(byte[] bytes, int offset) {
// copy identity hash
System.arraycopy(this.identityHash, 0, bytes, offset, OptOutConst.Sha256Bytes);
offset += OptOutConst.Sha256Bytes;

final byte metadata = calcMetadata(this.advertisingId);
final byte metadata = calcMetadata(this.advertisingId, isTombstone);

// copy advertising id
System.arraycopy(this.advertisingId, metadata == 0 ? 0 : 1, bytes, offset, OptOutConst.Sha256Bytes);
Expand All @@ -153,11 +179,16 @@ public void copyToByteArray(byte[] bytes, int offset) {
bytes[offset + Long.BYTES - 1] = metadata;
}

@Deprecated
public static void writeTo(ByteBuffer buffer, byte[] identityHash, byte[] advertisingId, long timestamp) {
writeTo(buffer, identityHash, advertisingId, timestamp, false);
}

public static void writeTo(ByteBuffer buffer, byte[] identityHash, byte[] advertisingId, long timestamp, boolean isTombstone) {
assert identityHash.length == OptOutConst.Sha256Bytes;
assert advertisingId.length == OptOutConst.Sha256Bytes || advertisingId.length == OptOutConst.Sha256Bytes + 1;

final byte metadata = calcMetadata(advertisingId);
final byte metadata = calcMetadata(advertisingId, isTombstone);
final byte[] timestampBytes = OptOutUtils.toByteArray(timestamp);
timestampBytes[timestampBytes.length - 1] = metadata;

Expand Down
9 changes: 9 additions & 0 deletions src/main/java/com/uid2/shared/optout/OptOutPartition.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,11 @@ public long getOptOutTimestamp(byte[] identityHash) {
return -1;
}

if (getTombstoneByIndex(entryIndex)) {
// this user optout has been marked for deletion (user opt-in)
return -1;
}

return getTimestampByIndex(entryIndex);
}

Expand Down Expand Up @@ -64,4 +69,8 @@ private long getTimestampByIndex(int entryIndex) {
// start byte index is calculated from itemIndex and optout entry size
return OptOutEntry.parseTimestamp(this.store, entryIndex * OptOutConst.EntrySize);
}

private boolean getTombstoneByIndex(int entryIndex) {
return OptOutEntry.parseTombstone(this.store, entryIndex * OptOutConst.EntrySize);
}
}
38 changes: 35 additions & 3 deletions src/test/java/com/uid2/shared/optout/OptOutEntryTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ public void parseLegacyEntry() {
Assert.assertArrayEquals(idHash, entry.identityHash);
Assert.assertArrayEquals(adsId, entry.advertisingId);
Assert.assertEquals(0x56555453525150l, entry.timestamp);
Assert.assertFalse(entry.isTombstone);
}

@Test
Expand All @@ -45,6 +46,7 @@ public void parseEntry() {
Assert.assertArrayEquals(idHash, entry.identityHash);
Assert.assertArrayEquals(expectedAdsId, entry.advertisingId);
Assert.assertEquals(0x56555453525150l, entry.timestamp);
Assert.assertFalse(entry.isTombstone);
}

@Test
Expand All @@ -69,6 +71,7 @@ public void parseEntryAtOffset() {
Assert.assertArrayEquals(idHash, entry.identityHash);
Assert.assertArrayEquals(expectedAdsId, entry.advertisingId);
Assert.assertEquals(0x56555453525150l, entry.timestamp);
Assert.assertFalse(entry.isTombstone);
}

@Test
Expand Down Expand Up @@ -113,6 +116,7 @@ public void setTimestampAtOffset() {
Assert.assertArrayEquals(idHash, entry.identityHash);
Assert.assertArrayEquals(expectedAdsId, entry.advertisingId);
Assert.assertEquals(newTimestamp, entry.timestamp);
Assert.assertFalse(entry.isTombstone);
}

@Test
Expand All @@ -124,13 +128,14 @@ public void copyToByteArrayLegacy()

final int offset = 12;
final byte[] records = new byte[offset + OptOutConst.EntrySize];
final OptOutEntry input = new OptOutEntry(idHash, adsId, timestamp);
final OptOutEntry input = new OptOutEntry(idHash, adsId, timestamp, false);
input.copyToByteArray(records, offset);

final OptOutEntry entry = OptOutEntry.parse(records, 12);
Assert.assertArrayEquals(idHash, entry.identityHash);
Assert.assertArrayEquals(adsId, entry.advertisingId);
Assert.assertEquals(0x56555453525150l, entry.timestamp);
Assert.assertFalse(entry.isTombstone);
}

@Test
Expand All @@ -142,13 +147,14 @@ public void copyToByteArray()

final int offset = 12;
final byte[] records = new byte[offset + OptOutConst.EntrySize];
final OptOutEntry input = new OptOutEntry(idHash, adsId, timestamp);
final OptOutEntry input = new OptOutEntry(idHash, adsId, timestamp, false);
input.copyToByteArray(records, offset);

final OptOutEntry entry = OptOutEntry.parse(records, 12);
Assert.assertArrayEquals(idHash, entry.identityHash);
Assert.assertArrayEquals(adsId, entry.advertisingId);
Assert.assertEquals(0x56555453525150l, entry.timestamp);
Assert.assertFalse(entry.isTombstone);
}

@Test
Expand All @@ -159,11 +165,37 @@ public void writeToByteBuffer()
final long timestamp = 0x56555453525150l;

ByteBuffer buffer = ByteBuffer.allocate(OptOutConst.EntrySize);
OptOutEntry.writeTo(buffer, idHash, adsId, timestamp);
OptOutEntry.writeTo(buffer, idHash, adsId, timestamp, false);

final OptOutEntry entry = OptOutEntry.parse(buffer.array(), 0);
Assert.assertArrayEquals(idHash, entry.identityHash);
Assert.assertArrayEquals(adsId, entry.advertisingId);
Assert.assertEquals(0x56555453525150l, entry.timestamp);
Assert.assertFalse(entry.isTombstone);
}

@Test
public void checkTombstone() {
final int offset = 12;
final byte[] idHash = OptOutUtils.hexToByteArray("101112131415161718191a1b1c1d1e1f202122232425262728292a2b2c2d2e2f");
final byte[] adsId = OptOutUtils.hexToByteArray("303132333435363738393a3b3c3d3e3f404142434445464748494a4b4c4d4e4f");
final byte[] timestamp = OptOutUtils.hexToByteArray("50515253545556");
final byte metadata = 0x74;

final byte[] records = new byte[offset + OptOutConst.EntrySize];
System.arraycopy(idHash, 0, records, offset, OptOutConst.Sha256Bytes);
System.arraycopy(adsId, 0, records, offset + OptOutConst.Sha256Bytes, OptOutConst.Sha256Bytes);
System.arraycopy(timestamp, 0, records, offset + OptOutConst.Sha256Bytes * 2, Long.BYTES - 1);
records[offset + OptOutConst.EntrySize - 1] = metadata;

final byte[] expectedAdsId = new byte[33];
expectedAdsId[0] = 0x14;
System.arraycopy(adsId, 0, expectedAdsId, 1, OptOutConst.Sha256Bytes);

final OptOutEntry entry = OptOutEntry.parse(records, offset);
Assert.assertArrayEquals(idHash, entry.identityHash);
Assert.assertArrayEquals(expectedAdsId, entry.advertisingId);
Assert.assertEquals(0x56555453525150l, entry.timestamp);
Assert.assertTrue(entry.isTombstone);
}
}