diff --git a/src/main/java/com/uid2/shared/optout/OptOutEntry.java b/src/main/java/com/uid2/shared/optout/OptOutEntry.java index 89ca54b8..9d923c91 100644 --- a/src/main/java/com/uid2/shared/optout/OptOutEntry.java +++ b/src/main/java/com/uid2/shared/optout/OptOutEntry.java @@ -12,7 +12,8 @@ // Total: 72 bytes // // Metadata format: -// Lower 6 bits -- identity type +// Lower 5 bits -- identity type +// Middle 1 bit -- tombstone // Higher 2 bits -- record version // Total: 1 byte // @@ -23,11 +24,15 @@ public final class OptOutEntry { public final byte[] identityHash; public final byte[] advertisingId; public final long timestamp; + public final boolean isTombstone; - private static final long timestampMask = 0xFFFFFFFFFFFFFFl; - private static final byte adsIdTypeMask = 0x3F; + private static final long timestampMask = 0xFFFFFFFFFFFFFFL; + private static final byte adsIdTypeMask = 0x1F; private static final int adsIdVersionShift = 6; + private static final byte tombstoneMask = 0x1; + private static final int tombstoneShift = 5; + @Deprecated public OptOutEntry(byte[] identityHash, byte[] advertisingId, long ts) { assert identityHash.length == OptOutConst.Sha256Bytes; assert advertisingId.length == OptOutConst.Sha256Bytes || advertisingId.length == OptOutConst.Sha256Bytes + 1; @@ -35,6 +40,17 @@ public OptOutEntry(byte[] identityHash, byte[] advertisingId, long ts) { this.identityHash = identityHash; this.advertisingId = advertisingId; this.timestamp = ts; + this.isTombstone = false; + } + + public OptOutEntry(byte[] identityHash, byte[] advertisingId, long ts, boolean isTombstone) { + assert identityHash.length == OptOutConst.Sha256Bytes; + assert advertisingId.length == OptOutConst.Sha256Bytes || advertisingId.length == OptOutConst.Sha256Bytes + 1; + // assert ts >= 0; + this.identityHash = identityHash; + this.advertisingId = advertisingId; + this.timestamp = ts; + this.isTombstone = isTombstone; } public static OptOutEntry parse(byte[] buffer, int bufferIndex) { @@ -43,6 +59,7 @@ public static OptOutEntry parse(byte[] buffer, int bufferIndex) { final byte metadata = buffer[bufferIndex + OptOutConst.EntrySize - 1]; final byte adsIdVersion = (byte) (metadata >> adsIdVersionShift); final byte identityType = (byte) (metadata & adsIdTypeMask); + final boolean isTombstone = (((byte) (metadata >> tombstoneShift)) & tombstoneMask) == 0x1; final byte[] idHash = Arrays.copyOfRange(buffer, bufferIndex, bufferIndex + OptOutConst.Sha256Bytes); bufferIndex += OptOutConst.Sha256Bytes; @@ -54,7 +71,7 @@ public static OptOutEntry parse(byte[] buffer, int bufferIndex) { final long ts = ByteBuffer.wrap(buffer, bufferIndex, Long.BYTES).order(ByteOrder.LITTLE_ENDIAN).getLong() & timestampMask; - return new OptOutEntry(idHash, adsId, ts); + return new OptOutEntry(idHash, adsId, ts, isTombstone); } private static byte[] parseAdsIdV3(byte[] buffer, int bufferIndex, byte identityType) { @@ -70,6 +87,12 @@ public static long parseTimestamp(byte[] buffer, int bufferIndexForEntry) { .order(ByteOrder.LITTLE_ENDIAN).getLong() & timestampMask; } + public static boolean parseTombstone(byte[] buffer, int bufferIndexForEntry) { + assert bufferIndexForEntry + OptOutConst.EntrySize <= buffer.length; + byte metadata = buffer[bufferIndexForEntry + (OptOutConst.Sha256Bytes << 1) + 7]; + return ((byte)(metadata >> tombstoneShift) & tombstoneMask) == (byte)0x1; + } + public static void setTimestamp(byte[] buffer, int bufferIndexForEntry, long timestamp) { assert bufferIndexForEntry + OptOutConst.EntrySize <= buffer.length; final byte metadata = buffer[bufferIndexForEntry + OptOutConst.EntrySize - 1]; @@ -109,7 +132,7 @@ public static OptOutEntry newRandom() { public static OptOutEntry newTestEntry(long idHash, long timestamp) { // for test, using the same value for identity_hash and advertising_id byte[] id = idHashFromLong(idHash); - return new OptOutEntry(id, id, timestamp); + return new OptOutEntry(id, id, timestamp, false); } // Overriding equals() to compare two OptOutEntry objects @@ -131,8 +154,11 @@ public int hashCode() { return (int) (timestamp + Arrays.hashCode(identityHash) + Arrays.hashCode(advertisingId)); } - private static byte calcMetadata(byte[] advertisingId) { - return (byte) (advertisingId.length == OptOutConst.Sha256Bytes ? 0 : ((1 << adsIdVersionShift) | advertisingId[0])); + private static byte calcMetadata(byte[] advertisingId, boolean isTombstone) { + return (byte) ( + (advertisingId.length == OptOutConst.Sha256Bytes ? 0 : ((1 << adsIdVersionShift) | advertisingId[0])) + | (isTombstone ? (1 << tombstoneShift) : 0) + ); } public void copyToByteArray(byte[] bytes, int offset) { @@ -140,7 +166,7 @@ public void copyToByteArray(byte[] bytes, int offset) { System.arraycopy(this.identityHash, 0, bytes, offset, OptOutConst.Sha256Bytes); offset += OptOutConst.Sha256Bytes; - final byte metadata = calcMetadata(this.advertisingId); + final byte metadata = calcMetadata(this.advertisingId, isTombstone); // copy advertising id System.arraycopy(this.advertisingId, metadata == 0 ? 0 : 1, bytes, offset, OptOutConst.Sha256Bytes); @@ -153,11 +179,16 @@ public void copyToByteArray(byte[] bytes, int offset) { bytes[offset + Long.BYTES - 1] = metadata; } + @Deprecated public static void writeTo(ByteBuffer buffer, byte[] identityHash, byte[] advertisingId, long timestamp) { + writeTo(buffer, identityHash, advertisingId, timestamp, false); + } + + public static void writeTo(ByteBuffer buffer, byte[] identityHash, byte[] advertisingId, long timestamp, boolean isTombstone) { assert identityHash.length == OptOutConst.Sha256Bytes; assert advertisingId.length == OptOutConst.Sha256Bytes || advertisingId.length == OptOutConst.Sha256Bytes + 1; - final byte metadata = calcMetadata(advertisingId); + final byte metadata = calcMetadata(advertisingId, isTombstone); final byte[] timestampBytes = OptOutUtils.toByteArray(timestamp); timestampBytes[timestampBytes.length - 1] = metadata; diff --git a/src/main/java/com/uid2/shared/optout/OptOutPartition.java b/src/main/java/com/uid2/shared/optout/OptOutPartition.java index 783aa0bf..e4c2eb02 100644 --- a/src/main/java/com/uid2/shared/optout/OptOutPartition.java +++ b/src/main/java/com/uid2/shared/optout/OptOutPartition.java @@ -21,6 +21,11 @@ public long getOptOutTimestamp(byte[] identityHash) { return -1; } + if (getTombstoneByIndex(entryIndex)) { + // this user optout has been marked for deletion (user opt-in) + return -1; + } + return getTimestampByIndex(entryIndex); } @@ -64,4 +69,8 @@ private long getTimestampByIndex(int entryIndex) { // start byte index is calculated from itemIndex and optout entry size return OptOutEntry.parseTimestamp(this.store, entryIndex * OptOutConst.EntrySize); } + + private boolean getTombstoneByIndex(int entryIndex) { + return OptOutEntry.parseTombstone(this.store, entryIndex * OptOutConst.EntrySize); + } } diff --git a/src/test/java/com/uid2/shared/optout/OptOutEntryTest.java b/src/test/java/com/uid2/shared/optout/OptOutEntryTest.java index 551cb7b7..5cb40c91 100644 --- a/src/test/java/com/uid2/shared/optout/OptOutEntryTest.java +++ b/src/test/java/com/uid2/shared/optout/OptOutEntryTest.java @@ -22,6 +22,7 @@ public void parseLegacyEntry() { Assert.assertArrayEquals(idHash, entry.identityHash); Assert.assertArrayEquals(adsId, entry.advertisingId); Assert.assertEquals(0x56555453525150l, entry.timestamp); + Assert.assertFalse(entry.isTombstone); } @Test @@ -45,6 +46,7 @@ public void parseEntry() { Assert.assertArrayEquals(idHash, entry.identityHash); Assert.assertArrayEquals(expectedAdsId, entry.advertisingId); Assert.assertEquals(0x56555453525150l, entry.timestamp); + Assert.assertFalse(entry.isTombstone); } @Test @@ -69,6 +71,7 @@ public void parseEntryAtOffset() { Assert.assertArrayEquals(idHash, entry.identityHash); Assert.assertArrayEquals(expectedAdsId, entry.advertisingId); Assert.assertEquals(0x56555453525150l, entry.timestamp); + Assert.assertFalse(entry.isTombstone); } @Test @@ -113,6 +116,7 @@ public void setTimestampAtOffset() { Assert.assertArrayEquals(idHash, entry.identityHash); Assert.assertArrayEquals(expectedAdsId, entry.advertisingId); Assert.assertEquals(newTimestamp, entry.timestamp); + Assert.assertFalse(entry.isTombstone); } @Test @@ -124,13 +128,14 @@ public void copyToByteArrayLegacy() final int offset = 12; final byte[] records = new byte[offset + OptOutConst.EntrySize]; - final OptOutEntry input = new OptOutEntry(idHash, adsId, timestamp); + final OptOutEntry input = new OptOutEntry(idHash, adsId, timestamp, false); input.copyToByteArray(records, offset); final OptOutEntry entry = OptOutEntry.parse(records, 12); Assert.assertArrayEquals(idHash, entry.identityHash); Assert.assertArrayEquals(adsId, entry.advertisingId); Assert.assertEquals(0x56555453525150l, entry.timestamp); + Assert.assertFalse(entry.isTombstone); } @Test @@ -142,13 +147,14 @@ public void copyToByteArray() final int offset = 12; final byte[] records = new byte[offset + OptOutConst.EntrySize]; - final OptOutEntry input = new OptOutEntry(idHash, adsId, timestamp); + final OptOutEntry input = new OptOutEntry(idHash, adsId, timestamp, false); input.copyToByteArray(records, offset); final OptOutEntry entry = OptOutEntry.parse(records, 12); Assert.assertArrayEquals(idHash, entry.identityHash); Assert.assertArrayEquals(adsId, entry.advertisingId); Assert.assertEquals(0x56555453525150l, entry.timestamp); + Assert.assertFalse(entry.isTombstone); } @Test @@ -159,11 +165,37 @@ public void writeToByteBuffer() final long timestamp = 0x56555453525150l; ByteBuffer buffer = ByteBuffer.allocate(OptOutConst.EntrySize); - OptOutEntry.writeTo(buffer, idHash, adsId, timestamp); + OptOutEntry.writeTo(buffer, idHash, adsId, timestamp, false); final OptOutEntry entry = OptOutEntry.parse(buffer.array(), 0); Assert.assertArrayEquals(idHash, entry.identityHash); Assert.assertArrayEquals(adsId, entry.advertisingId); Assert.assertEquals(0x56555453525150l, entry.timestamp); + Assert.assertFalse(entry.isTombstone); + } + + @Test + public void checkTombstone() { + final int offset = 12; + final byte[] idHash = OptOutUtils.hexToByteArray("101112131415161718191a1b1c1d1e1f202122232425262728292a2b2c2d2e2f"); + final byte[] adsId = OptOutUtils.hexToByteArray("303132333435363738393a3b3c3d3e3f404142434445464748494a4b4c4d4e4f"); + final byte[] timestamp = OptOutUtils.hexToByteArray("50515253545556"); + final byte metadata = 0x74; + + final byte[] records = new byte[offset + OptOutConst.EntrySize]; + System.arraycopy(idHash, 0, records, offset, OptOutConst.Sha256Bytes); + System.arraycopy(adsId, 0, records, offset + OptOutConst.Sha256Bytes, OptOutConst.Sha256Bytes); + System.arraycopy(timestamp, 0, records, offset + OptOutConst.Sha256Bytes * 2, Long.BYTES - 1); + records[offset + OptOutConst.EntrySize - 1] = metadata; + + final byte[] expectedAdsId = new byte[33]; + expectedAdsId[0] = 0x14; + System.arraycopy(adsId, 0, expectedAdsId, 1, OptOutConst.Sha256Bytes); + + final OptOutEntry entry = OptOutEntry.parse(records, offset); + Assert.assertArrayEquals(idHash, entry.identityHash); + Assert.assertArrayEquals(expectedAdsId, entry.advertisingId); + Assert.assertEquals(0x56555453525150l, entry.timestamp); + Assert.assertTrue(entry.isTombstone); } }