Skip to content
This repository has been archived by the owner on Feb 12, 2022. It is now read-only.

Master apache #719

Open
wants to merge 4 commits into
base: master-Apache
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 13 additions & 5 deletions phoenix-core/src/main/antlr3/PhoenixSQL.g
Original file line number Diff line number Diff line change
Expand Up @@ -521,11 +521,19 @@ dyn_column_def returns [ColumnDef ret]

dyn_column_name_or_def returns [ColumnDef ret]
: c=column_name (dt=identifier (LPAREN l=NUMBER (COMMA s=NUMBER)? RPAREN)? )? (lsq=LSQUARE (a=NUMBER)? RSQUARE)?
{$ret = factory.columnDef(c, dt, true,
l == null ? null : Integer.parseInt( l.getText() ),
s == null ? null : Integer.parseInt( s.getText() ),
false,
null); }
{
if(lsq != null)
{
throwRecognitionException(lsq);
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If this is always an error, then we wouldn't want it in the syntax, but I'm actually capturing this information now and storing it in an ARRAY_SIZE key value column on the column metadata row. We're not using it, but we should use it down the road.

With our dynamic column feature, you can define a column at upsert time and populate it. This would include arrays. I think for arrays, you should never have a null dt. I think it's just a matter of not having both the dt part and the array brackets be optional - instead the whole thing should be optional.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

So for now I will revert this change? I was just wondering how that
bq.(lsq=LSQUARE (a=NUMBER)? RSQUARE)?
was added. I was thinking I added that in my previous commit. So you don't want that to be solved in the syntax level? Ok will see then what to do.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Make sure you switch to the Apache git repo as the github repo is no longer
being used - your on the phoenix dev list, right and have been getting my
mail? If not please join here:
http://phoenix.incubator.apache.org/mailing_list.html

Get the latest from the Apache git repo and you'll see my fix - it was just
modifying the syntax slightly.

For PDataType.getByteSize() should return null if the type doesn't know how
big it is, which is fine for arrays. For the Expression (i.e. this would be
a KeyValueColumnExpression specifically), the getByteSize() has all the
information it needs to calculate the size correctly.

} else
{
$ret = factory.columnDef(c, dt, true,
l == null ? null : Integer.parseInt( l.getText() ),
s == null ? null : Integer.parseInt( s.getText() ),
false,
null);
}
}
;

select_expression returns [SelectStatement ret]
Expand Down
138 changes: 80 additions & 58 deletions phoenix-core/src/main/java/org/apache/phoenix/parse/ColumnDef.java
Original file line number Diff line number Diff line change
Expand Up @@ -66,64 +66,86 @@ public class ColumnDef {
}

this.isNull = isNull;
if (this.dataType == PDataType.CHAR) {
if (maxLength == null) {
throw new SQLExceptionInfo.Builder(SQLExceptionCode.MISSING_CHAR_LENGTH)
.setColumnName(columnDefName.getColumnName()).build().buildException();
}
if (maxLength < 1) {
throw new SQLExceptionInfo.Builder(SQLExceptionCode.NONPOSITIVE_CHAR_LENGTH)
.setColumnName(columnDefName.getColumnName()).build().buildException();
}
scale = null;
} else if (this.dataType == PDataType.VARCHAR) {
if (maxLength != null && maxLength < 1) {
throw new SQLExceptionInfo.Builder(SQLExceptionCode.NONPOSITIVE_CHAR_LENGTH)
.setColumnName(columnDefName.getColumnName()).build().buildException();
}
scale = null;
} else if (this.dataType == PDataType.DECIMAL) {
Integer origMaxLength = maxLength;
maxLength = maxLength == null ? PDataType.MAX_PRECISION : maxLength;
// for deciaml, 1 <= maxLength <= PDataType.MAX_PRECISION;
if (maxLength < 1 || maxLength > PDataType.MAX_PRECISION) {
throw new SQLExceptionInfo.Builder(SQLExceptionCode.DECIMAL_PRECISION_OUT_OF_RANGE)
.setColumnName(columnDefName.getColumnName()).build().buildException();
}
// When a precision is specified and a scale is not specified, it is set to 0.
//
// This is the standard as specified in
// http://docs.oracle.com/cd/B28359_01/server.111/b28318/datatype.htm#CNCPT1832
// and
// http://docs.oracle.com/javadb/10.6.2.1/ref/rrefsqlj15260.html.
// Otherwise, if scale is bigger than maxLength, just set it to the maxLength;
//
// When neither a precision nor a scale is specified, the precision and scale is
// ignored. All decimal are stored with as much decimal points as possible.
scale = scale == null ?
origMaxLength == null ? null : PDataType.DEFAULT_SCALE :
scale > maxLength ? maxLength : scale;
} else if (this.dataType == PDataType.BINARY) {
if (maxLength == null) {
throw new SQLExceptionInfo.Builder(SQLExceptionCode.MISSING_BINARY_LENGTH)
.setColumnName(columnDefName.getColumnName()).build().buildException();
}
if (maxLength < 1) {
throw new SQLExceptionInfo.Builder(SQLExceptionCode.NONPOSITIVE_BINARY_LENGTH)
.setColumnName(columnDefName.getColumnName()).build().buildException();
}
scale = null;
} else if (this.dataType == PDataType.INTEGER) {
maxLength = PDataType.INT_PRECISION;
scale = PDataType.ZERO;
} else if (this.dataType == PDataType.LONG) {
maxLength = PDataType.LONG_PRECISION;
scale = PDataType.ZERO;
} else {
// ignore maxLength and scale for other types.
maxLength = null;
scale = null;
}
if (!this.isArray) {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This would be bad. We need the maxLength for arrays too. It represents the maxLength of the element (i.e. the precision for a DECIMAL or the max length for a CHAR).

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

As i am returning getByteSize as null, this needs to be changed or add
/**
* Estimate the byte size from the type length. For example, for char, byte size would be the
* same as length. For decimal, byte size would have no correlation with the length.
*/
public Integer estimateByteSizeFromLength(Integer length) {
if (isFixedWidth()) {
return getByteSize();
}
if(isArrayType()) {
return null;
}
// If not fixed width, default to say the byte size is the same as length.
return length;
}

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'd recommend working on #669. I need to do a bit of clean up (there's
definitely at least on too many getByteSize methods...).

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Got the code from the apache git repo now.
I saw that

         if(isArrayType()) {
        return null;
    }

this has been left as it is. If you are ok with this then am fine. Will start on #669

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

That's probably a good fix to make. You have the length passed into that
function, so calculating the size if it's an array is probably pretty
straightforward (famous last words :-).

On Mon, Jan 27, 2014 at 11:21 PM, ramkrish86 [email protected]:

In phoenix-core/src/main/java/org/apache/phoenix/parse/ColumnDef.java:

  •             throw new SQLExceptionInfo.Builder(SQLExceptionCode.NONPOSITIVE_BINARY_LENGTH)
    
  •                 .setColumnName(columnDefName.getColumnName()).build().buildException();
    
  •         }
    
  •         scale = null;
    
  •     } else if (this.dataType == PDataType.INTEGER) {
    
  •         maxLength = PDataType.INT_PRECISION;
    
  •         scale = PDataType.ZERO;
    
  •     } else if (this.dataType == PDataType.LONG) {
    
  •         maxLength = PDataType.LONG_PRECISION;
    
  •         scale = PDataType.ZERO;
    
  •     } else {
    
  •         // ignore maxLength and scale for other types.
    
  •         maxLength = null;
    
  •         scale = null;
    
  •     }
    
  •       if (!this.isArray) {
    

Got the code from the apache git repo now.

I saw that

     if(isArrayType()) {
    return null;
}

this has been left as it is. If you are ok with this then am fine. Will
start on #669 #669

Reply to this email directly or view it on GitHubhttps://github.com//pull/719/files#r9219300
.

if (this.dataType == PDataType.CHAR) {
if (maxLength == null) {
throw new SQLExceptionInfo.Builder(
SQLExceptionCode.MISSING_CHAR_LENGTH)
.setColumnName(columnDefName.getColumnName())
.build().buildException();
}
if (maxLength < 1) {
throw new SQLExceptionInfo.Builder(
SQLExceptionCode.NONPOSITIVE_CHAR_LENGTH)
.setColumnName(columnDefName.getColumnName())
.build().buildException();
}
scale = null;
} else if (this.dataType == PDataType.VARCHAR) {
if (maxLength != null && maxLength < 1) {
throw new SQLExceptionInfo.Builder(
SQLExceptionCode.NONPOSITIVE_CHAR_LENGTH)
.setColumnName(columnDefName.getColumnName())
.build().buildException();
}
scale = null;
} else if (this.dataType == PDataType.DECIMAL) {
Integer origMaxLength = maxLength;
maxLength = maxLength == null ? PDataType.MAX_PRECISION
: maxLength;
// for deciaml, 1 <= maxLength <= PDataType.MAX_PRECISION;
if (maxLength < 1 || maxLength > PDataType.MAX_PRECISION) {
throw new SQLExceptionInfo.Builder(
SQLExceptionCode.DECIMAL_PRECISION_OUT_OF_RANGE)
.setColumnName(columnDefName.getColumnName())
.build().buildException();
}
// When a precision is specified and a scale is not
// specified, it is set to 0.
//
// This is the standard as specified in
// http://docs.oracle.com/cd/B28359_01/server.111/b28318/datatype.htm#CNCPT1832
// and
// http://docs.oracle.com/javadb/10.6.2.1/ref/rrefsqlj15260.html.
// Otherwise, if scale is bigger than maxLength, just set it
// to the maxLength;
//
// When neither a precision nor a scale is specified, the
// precision and scale is
// ignored. All decimal are stored with as much decimal
// points as possible.
scale = scale == null ? origMaxLength == null ? null
: PDataType.DEFAULT_SCALE
: scale > maxLength ? maxLength : scale;
} else if (this.dataType == PDataType.BINARY) {
if (maxLength == null) {
throw new SQLExceptionInfo.Builder(
SQLExceptionCode.MISSING_BINARY_LENGTH)
.setColumnName(columnDefName.getColumnName())
.build().buildException();
}
if (maxLength < 1) {
throw new SQLExceptionInfo.Builder(
SQLExceptionCode.NONPOSITIVE_BINARY_LENGTH)
.setColumnName(columnDefName.getColumnName())
.build().buildException();
}
scale = null;
} else if (this.dataType == PDataType.INTEGER) {
maxLength = PDataType.INT_PRECISION;
scale = PDataType.ZERO;
} else if (this.dataType == PDataType.LONG) {
maxLength = PDataType.LONG_PRECISION;
scale = PDataType.ZERO;
} else {
// ignore maxLength and scale for other types.
maxLength = null;
scale = null;
}
} else {
maxLength = null;
scale = null;
}
this.maxLength = maxLength;
this.scale = scale;
this.isPK = isPK;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,32 +19,21 @@
*/
package org.apache.phoenix.schema;

import java.math.BigDecimal;
import java.math.BigInteger;
import java.math.MathContext;
import java.math.RoundingMode;
import java.sql.Date;
import java.sql.Time;
import java.sql.Timestamp;
import java.sql.Types;
import java.math.*;
import java.sql.*;
import java.text.Format;
import java.util.Map;

import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.hbase.util.Base64;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.phoenix.query.KeyRange;
import org.apache.phoenix.query.QueryConstants;
import org.apache.phoenix.util.*;

import com.google.common.collect.ImmutableMap;
import com.google.common.math.LongMath;
import com.google.common.primitives.Booleans;
import com.google.common.primitives.Doubles;
import com.google.common.primitives.Longs;
import org.apache.phoenix.query.KeyRange;
import org.apache.phoenix.query.QueryConstants;
import org.apache.phoenix.util.ByteUtil;
import org.apache.phoenix.util.DateUtil;
import org.apache.phoenix.util.NumberUtil;
import org.apache.phoenix.util.StringUtil;
import com.google.common.primitives.*;


/**
Expand Down Expand Up @@ -5193,9 +5182,6 @@ public Integer estimateByteSizeFromLength(Integer length) {
if (isFixedWidth()) {
return getByteSize();
}
if(isArrayType()) {
return null;
}
// If not fixed width, default to say the byte size is the same as length.
return length;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,31 +19,18 @@
*/
package org.apache.phoenix.end2end;

import static org.apache.phoenix.util.TestUtil.B_VALUE;
import static org.apache.phoenix.util.TestUtil.PHOENIX_JDBC_URL;
import static org.apache.phoenix.util.TestUtil.ROW1;
import static org.apache.phoenix.util.TestUtil.TABLE_WITH_ARRAY;
import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
import static org.apache.phoenix.util.TestUtil.*;
import static org.junit.Assert.*;

import java.sql.Array;
import java.sql.Connection;
import java.sql.Date;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.*;
import java.util.Properties;

import org.apache.phoenix.exception.PhoenixParserException;
import org.apache.phoenix.query.BaseTest;
import org.apache.phoenix.schema.PhoenixArray;
import org.apache.phoenix.util.PhoenixRuntime;
import org.apache.phoenix.util.StringUtil;
import org.junit.Assert;
import org.junit.Ignore;
import org.junit.Test;

import com.google.common.primitives.Floats;
Expand Down Expand Up @@ -438,50 +425,34 @@ public void testUpsertSelectWithColumnRef() throws Exception {
}
}

@Ignore //TODO: Ram to fix
@Test
public void testUpsertSelectWithSelectAsSubQuery3() throws Exception {
long ts = nextTimestamp();
String tenantId = getOrganizationId();
createTableWithArray(BaseConnectedQueryTest.getUrl(),
getDefaultSplits(tenantId), null, ts - 2);
//initTablesWithArrays(tenantId, null, ts, false);
try {
createSimpleTableWithArray(BaseConnectedQueryTest.getUrl(),
getDefaultSplits(tenantId), null, ts - 2);
initSimpleArrayTable(tenantId, null, ts, false);
Properties props = new Properties(TEST_PROPERTIES);
props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB,
Long.toString(ts + 2)); // Execute at timestamp 2
Connection conn = DriverManager.getConnection(PHOENIX_JDBC_URL,
props);
// TODO: this is invalid, as you can't have an array reference in upsert
String query = "upsert into table_with_array(ORGANIZATION_ID,ENTITY_ID,a_double_array[3]) values('"
+ tenantId + "','00A123122312312',2.0d)";
PreparedStatement statement = conn.prepareStatement(query);
int executeUpdate = statement.executeUpdate();
assertEquals(1, executeUpdate);
conn.commit();
statement.close();
conn.close();
// create another connection
props = new Properties(TEST_PROPERTIES);
props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB,
Long.toString(ts + 4));
conn = DriverManager.getConnection(PHOENIX_JDBC_URL, props);
query = "SELECT ARRAY_ELEM(a_double_array,3) FROM table_with_array";
statement = conn.prepareStatement(query);
ResultSet rs = statement.executeQuery();
assertTrue(rs.next());
// Need to support primitive
Double[] doubleArr = new Double[1];
doubleArr[0] = 2.0d;
conn.createArrayOf("DOUBLE", doubleArr);
Double result = rs.getDouble(1);
assertEquals(result, doubleArr[0]);

} finally {
}
// TODO : currently not supported
public void testUpsertSelectWithSpecificIndexOfAnArray() throws Exception {
long ts = nextTimestamp();
String tenantId = getOrganizationId();
createTableWithArray(BaseConnectedQueryTest.getUrl(),
getDefaultSplits(tenantId), null, ts - 2);
// initTablesWithArrays(tenantId, null, ts, false);
Connection conn = null;
try {
createSimpleTableWithArray(BaseConnectedQueryTest.getUrl(),
getDefaultSplits(tenantId), null, ts - 2);
initSimpleArrayTable(tenantId, null, ts, false);
Properties props = new Properties(TEST_PROPERTIES);
props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB,
Long.toString(ts + 2)); // Execute at timestamp 2
conn = DriverManager.getConnection(PHOENIX_JDBC_URL, props);
// TODO: this is invalid, as you can't have an array reference in
// upsert
String query = "upsert into table_with_array(ORGANIZATION_ID,ENTITY_ID,a_double_array[3]) values('"
+ tenantId + "','00A123122312312',2.0d)";
PreparedStatement statement = conn.prepareStatement(query);
fail("Should have failed with parser Exception");
} catch (PhoenixParserException e) {
// Correct
} finally {
conn.close();
}
}

@Test
Expand Down