Skip to content

Fix reverted AttributeTypeUtils #3127

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Yicong-Huang merged 4 commits into from
Dec 2, 2024
Merged

Fix reverted AttributeTypeUtils #3127

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
067cc42
apply fix
Yicong-Huang Nov 29, 2024
fb3fff2
fix format
Yicong-Huang Nov 29, 2024
f144e08
Merge branch 'master' into yicong-fix-type-casting
Yicong-Huang Nov 29, 2024
10521b0
Merge branch 'master' into yicong-fix-type-casting
Yicong-Huang Dec 2, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
192 changes: 129 additions & 63 deletions core/workflow-core/src/main/scala/edu/uci/ics/amber/core/tuple/AttributeTypeUtils.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,8 @@ package edu.uci.ics.amber.core.tuple
import com.github.sisyphsu.dateparser.DateParserUtils

import java.sql.Timestamp
import java.text.NumberFormat
import java.util.Locale
import scala.util.Try
import scala.util.control.Exception.allCatch

Expand Down Expand Up @@ -64,7 +66,7 @@ object AttributeTypeUtils extends Serializable {
TupleLike(
tuple.getSchema.getAttributes.map { attr =>
val targetType = targetTypes.getOrElse(attr.getName, attr.getType)
parseField(tuple.getField(attr.getName), targetType)
parseField(tuple.getField(attr.getName), targetType, force = true)
}
)

Expand All @@ -90,18 +92,21 @@ object AttributeTypeUtils extends Serializable {
* parse Field to a corresponding Java object base on the given Schema AttributeType
* @param field fields value
* @param attributeType target AttributeType
* @param force force to parse the field to the target type if possible
* currently only support for comma-separated numbers
*
* @return parsedField in the target AttributeType
*/
@throws[AttributeTypeException]
def parseField(
field: Any,
attributeType: AttributeType
attributeType: AttributeType,
force: Boolean = false
): Any = {
if (field == null) return null
attributeType match {
case AttributeType.INTEGER => parseInteger(field)
case AttributeType.LONG => parseLong(field)
case AttributeType.INTEGER => parseInteger(field, force)
case AttributeType.LONG => parseLong(field, force)
case AttributeType.DOUBLE => parseDouble(field)
case AttributeType.BOOLEAN => parseBoolean(field)
case AttributeType.TIMESTAMP => parseTimestamp(field)
Expand All @@ -112,87 +117,147 @@ object AttributeTypeUtils extends Serializable {
}

@throws[AttributeTypeException]
private def parseInteger(fieldValue: Any): Integer = {
fieldValue match {
case str: String => str.trim.toInt
case int: Integer => int
case long: java.lang.Long => long.toInt
case double: java.lang.Double => double.toInt
case boolean: java.lang.Boolean => if (boolean) 1 else 0
// Timestamp and Binary are considered to be illegal here.
case _ =>
private def parseInteger(fieldValue: Any, force: Boolean = false): Integer = {
val attempt: Try[Integer] = Try {
fieldValue match {
case str: String =>
if (force) {
// Use US locale for comma-separated numbers
NumberFormat.getNumberInstance(Locale.US).parse(str.trim).intValue()
} else {
str.trim.toInt
}
case int: Integer => int
case long: java.lang.Long => long.toInt
case double: java.lang.Double => double.toInt
case boolean: java.lang.Boolean => if (boolean) 1 else 0
// Timestamp and Binary are considered to be illegal here.
case _ =>
throw new IllegalArgumentException(
s"Unsupported type for parsing to Integer: ${fieldValue.getClass.getName}"
)
}
}

attempt.recover {
case e: Exception =>
throw new AttributeTypeException(
s"not able to parse type ${fieldValue.getClass} to Integer: ${fieldValue.toString}"
s"Failed to parse type ${fieldValue.getClass.getName} to Integer: ${fieldValue.toString}",
e
)
}
}.get
}

@throws[AttributeTypeException]
private def parseLong(fieldValue: Any): java.lang.Long = {
fieldValue match {
case str: String => str.trim.toLong
case int: Integer => int.toLong
case long: java.lang.Long => long
case double: java.lang.Double => double.toLong
case boolean: java.lang.Boolean => if (boolean) 1L else 0L
case timestamp: Timestamp => timestamp.toInstant.toEpochMilli
// Binary is considered to be illegal here.
case _ =>
private def parseLong(fieldValue: Any, force: Boolean = false): java.lang.Long = {
val attempt: Try[Long] = Try {
fieldValue match {
case str: String =>
if (force) {
// Use US locale for comma-separated numbers
NumberFormat.getNumberInstance(Locale.US).parse(str.trim).longValue()
} else {
str.trim.toLong
}
case int: Integer => int.toLong
case long: java.lang.Long => long
case double: java.lang.Double => double.toLong
case boolean: java.lang.Boolean => if (boolean) 1L else 0L
case timestamp: Timestamp => timestamp.toInstant.toEpochMilli
// Binary is considered to be illegal here.
case _ =>
throw new IllegalArgumentException(
s"Unsupported type for parsing to Long: ${fieldValue.getClass.getName}"
)
}
}
attempt.recover {
case e: Exception =>
throw new AttributeTypeException(
s"not able to parse type ${fieldValue.getClass} to Long: ${fieldValue.toString}"
s"Failed to parse type ${fieldValue.getClass.getName} to Long: ${fieldValue.toString}",
e
)
}
}.get
}

@throws[AttributeTypeException]
def parseTimestamp(fieldValue: Any): Timestamp = {
val parseError = new AttributeTypeException(
s"not able to parse type ${fieldValue.getClass} to Timestamp: ${fieldValue.toString}"
)
fieldValue match {
case str: String => new Timestamp(DateParserUtils.parseDate(str.trim).getTime)
case long: java.lang.Long => new Timestamp(long)
case timestamp: Timestamp => timestamp
case date: java.util.Date => new Timestamp(date.getTime)
// Integer, Double, Boolean, Binary are considered to be illegal here.
case _ =>
throw parseError
val attempt: Try[Timestamp] = Try {
fieldValue match {
case str: String => new Timestamp(DateParserUtils.parseDate(str.trim).getTime)
case long: java.lang.Long => new Timestamp(long)
case timestamp: Timestamp => timestamp
case date: java.util.Date => new Timestamp(date.getTime)
// Integer, Double, Boolean, Binary are considered to be illegal here.
case _ =>
throw new AttributeTypeException(
s"Unsupported type for parsing to Timestamp: ${fieldValue.getClass.getName}"
)
}
}

attempt.recover {
case e: Exception =>
throw new AttributeTypeException(
s"Failed to parse type ${fieldValue.getClass.getName} to Timestamp: ${fieldValue.toString}",
e
)
}.get

}

@throws[AttributeTypeException]
def parseDouble(fieldValue: Any): java.lang.Double = {
fieldValue match {
case str: String => str.trim.toDouble
case int: Integer => int.toDouble
case long: java.lang.Long => long.toDouble
case double: java.lang.Double => double
case boolean: java.lang.Boolean => if (boolean) 1 else 0
// Timestamp and Binary are considered to be illegal here.
case _ =>
val attempt: Try[Double] = Try {
fieldValue match {
case str: String => str.trim.toDouble
case int: Integer => int.toDouble
case long: java.lang.Long => long.toDouble
case double: java.lang.Double => double
case boolean: java.lang.Boolean => if (boolean) 1 else 0
// Timestamp and Binary are considered to be illegal here.
case _ =>
throw new AttributeTypeException(
s"Unsupported type for parsing to Double: ${fieldValue.getClass.getName}"
)
}
}

attempt.recover {
case e: Exception =>
throw new AttributeTypeException(
s"not able to parse type ${fieldValue.getClass} to Double: ${fieldValue.toString}"
s"Failed to parse type ${fieldValue.getClass.getName} to Double: ${fieldValue.toString}",
e
)
}
}.get

}

@throws[AttributeTypeException]
private def parseBoolean(fieldValue: Any): java.lang.Boolean = {
val parseError = new AttributeTypeException(
s"not able to parse type ${fieldValue.getClass} to Boolean: ${fieldValue.toString}"
)
fieldValue match {
case str: String =>
(Try(str.trim.toBoolean) orElse Try(str.trim.toInt == 1))
.getOrElse(throw parseError)
case int: Integer => int != 0
case long: java.lang.Long => long != 0
case double: java.lang.Double => double != 0
case boolean: java.lang.Boolean => boolean
// Timestamp and Binary are considered to be illegal here.
case _ =>
throw parseError
val attempt: Try[Boolean] = Try {
fieldValue match {
case str: String =>
(Try(str.trim.toBoolean) orElse Try(str.trim.toInt == 1)).get
case int: Integer => int != 0
case long: java.lang.Long => long != 0
case double: java.lang.Double => double != 0
case boolean: java.lang.Boolean => boolean
// Timestamp and Binary are considered to be illegal here.
case _ =>
throw new AttributeTypeException(
s"Unsupported type for parsing to Boolean: ${fieldValue.getClass.getName}"
)
}
}

attempt.recover {
case e: Exception =>
throw new AttributeTypeException(
s"Failed to parse type ${fieldValue.getClass.getName} to Boolean: ${fieldValue.toString}",
e
)
}.get
}

/**
Expand Down Expand Up @@ -306,5 +371,6 @@ object AttributeTypeUtils extends Serializable {
}
}

class AttributeTypeException(msg: String) extends IllegalArgumentException(msg) {}
class AttributeTypeException(msg: String, cause: Throwable = null)
extends IllegalArgumentException(msg, cause) {}
}
90 changes: 89 additions & 1 deletion core/workflow-core/src/test/scala/edu/uci/ics/amber/core/tuple/AttributeTypeUtilsSpec.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,12 @@
package edu.uci.ics.amber.core.tuple

import edu.uci.ics.amber.core.tuple.AttributeTypeUtils.{inferField, inferSchemaFromRows}
import edu.uci.ics.amber.core.tuple.AttributeType._
import edu.uci.ics.amber.core.tuple.AttributeTypeUtils.{
AttributeTypeException,
inferField,
inferSchemaFromRows,
parseField
}
import org.scalatest.funsuite.AnyFunSuite

class AttributeTypeUtilsSpec extends AnyFunSuite {
Expand Down Expand Up @@ -83,4 +88,87 @@ class AttributeTypeUtilsSpec extends AnyFunSuite {

}

test("parseField correctly parses to INTEGER") {
assert(parseField("123", AttributeType.INTEGER) == 123)
assert(parseField("1,234", AttributeType.INTEGER, force = true) == 1234)
assert(parseField(456, AttributeType.INTEGER) == 456)
assert(parseField(123.45, AttributeType.INTEGER) == 123)
assert(parseField(true, AttributeType.INTEGER) == 1)
assert(parseField(false, AttributeType.INTEGER) == 0)
assertThrows[AttributeTypeException] {
parseField("invalid", AttributeType.INTEGER)
}
assertThrows[AttributeTypeException] {
parseField("1,234", AttributeType.INTEGER)
}
}

test("parseField correctly parses to LONG") {
assert(parseField("1234567890", AttributeType.LONG) == 1234567890L)
assert(parseField("1,234,567", AttributeType.LONG, force = true) == 1234567L)
assert(parseField(12345L, AttributeType.LONG) == 12345L)
assert(parseField(123.45, AttributeType.LONG) == 123L)
assert(parseField(true, AttributeType.LONG) == 1L)
assertThrows[AttributeTypeException] {
parseField("invalid", AttributeType.LONG)
}
assertThrows[AttributeTypeException] {
parseField("1,234,567", AttributeType.LONG)
}
}

test("parseField correctly parses to DOUBLE") {
assert(parseField("123.45", AttributeType.DOUBLE) == 123.45)
assert(parseField(12345, AttributeType.DOUBLE) == 12345.0)
assert(parseField(12345L, AttributeType.DOUBLE) == 12345.0)
assert(parseField(true, AttributeType.DOUBLE) == 1.0)
assertThrows[AttributeTypeException] {
parseField("invalid", AttributeType.DOUBLE)
}
}

test("parseField correctly parses to BOOLEAN") {
assert(parseField("true", AttributeType.BOOLEAN) == true)
assert(parseField("True", AttributeType.BOOLEAN) == true)
assert(parseField("TRUE", AttributeType.BOOLEAN) == true)
assert(parseField("false", AttributeType.BOOLEAN) == false)
assert(parseField("False", AttributeType.BOOLEAN) == false)
assert(parseField("FALSE", AttributeType.BOOLEAN) == false)
assert(parseField("1", AttributeType.BOOLEAN) == true)
assert(parseField("0", AttributeType.BOOLEAN) == false)
assert(parseField(1, AttributeType.BOOLEAN) == true)
assert(parseField(0, AttributeType.BOOLEAN) == false)
assertThrows[AttributeTypeException] {
parseField("invalid", AttributeType.BOOLEAN)
}
}

test("parseField correctly parses to TIMESTAMP") {
val timestamp =
parseField("2023-11-13T10:15:30", AttributeType.TIMESTAMP).asInstanceOf[java.sql.Timestamp]
assert(timestamp.toString == "2023-11-13 10:15:30.0")

assert(
parseField(1699820130000L, AttributeType.TIMESTAMP)
.asInstanceOf[java.sql.Timestamp]
.getTime == 1699820130000L
)

assertThrows[AttributeTypeException] {
parseField("invalid", AttributeType.TIMESTAMP)
}
}

test("parseField correctly parses to STRING") {
assert(parseField(123, AttributeType.STRING) == "123")
assert(parseField(123.45, AttributeType.STRING) == "123.45")
assert(parseField(true, AttributeType.STRING) == "true")
}

test("parseField returns original value for BINARY and ANY") {
val binaryData = Array[Byte](1, 2, 3)
assert(parseField(binaryData, AttributeType.BINARY) == binaryData)
assert(parseField("anything", AttributeType.ANY) == "anything")
}

}