diff --git a/WORKSPACE b/WORKSPACE
index 1b3893ef6..3fdefac1b 100644
--- a/WORKSPACE
+++ b/WORKSPACE
@@ -272,11 +272,11 @@ http_archive(
         """sed -i.bak 's/type_traits/std::max<int16_t>(sizeof(int16_t), type_traits/g' cpp/src/parquet/column_reader.cc""",
         """sed -i.bak 's/value_byte_size/value_byte_size)/g' cpp/src/parquet/column_reader.cc""",
     ],
-    sha256 = "a27971e2a71c412ae43d998b7b6d06201c7a3da382c804dcdc4a8126ccbabe67",
-    strip_prefix = "arrow-apache-arrow-4.0.0",
+    sha256 = "fc461c4f0a60e7470a7c58b28e9344aa8fb0be5cc982e9658970217e084c3a82",
+    strip_prefix = "arrow-apache-arrow-3.0.0",
     urls = [
-        "https://storage.googleapis.com/mirror.tensorflow.org/github.com/apache/arrow/archive/apache-arrow-4.0.0.tar.gz",
-        "https://github.com/apache/arrow/archive/apache-arrow-4.0.0.tar.gz",
+        "https://storage.googleapis.com/mirror.tensorflow.org/github.com/apache/arrow/archive/apache-arrow-3.0.0.tar.gz",
+        "https://github.com/apache/arrow/archive/apache-arrow-3.0.0.tar.gz",
     ],
 )
 
@@ -1135,13 +1135,3 @@ http_archive(
         "https://github.com/apache/orc/archive/refs/tags/rel/release-1.6.7.tar.gz",
     ],
 )
-
-http_archive(
-    name = "xsimd",
-    build_file = "//third_party:xsimd.BUILD",
-    sha256 = "45337317c7f238fe0d64bb5d5418d264a427efc53400ddf8e6a964b6bcb31ce9",
-    strip_prefix = "xsimd-7.5.0",
-    urls = [
-        "https://github.com/xtensor-stack/xsimd/archive/refs/tags/7.5.0.tar.gz",
-    ],
-)
diff --git a/tensorflow_io/core/kernels/csv_kernels.cc b/tensorflow_io/core/kernels/csv_kernels.cc
index c9303113c..196d1a31c 100644
--- a/tensorflow_io/core/kernels/csv_kernels.cc
+++ b/tensorflow_io/core/kernels/csv_kernels.cc
@@ -45,8 +45,8 @@ class CSVReadable : public IOReadableInterface {
     csv_file_.reset(new ArrowRandomAccessFile(file_.get(), file_size_));
 
     auto result = ::arrow::csv::TableReader::Make(
-        ::arrow::default_memory_pool(), ::arrow::io::default_io_context(),
-        csv_file_, ::arrow::csv::ReadOptions::Defaults(),
+        ::arrow::default_memory_pool(), csv_file_,
+        ::arrow::csv::ReadOptions::Defaults(),
         ::arrow::csv::ParseOptions::Defaults(),
         ::arrow::csv::ConvertOptions::Defaults());
     if (!result.status().ok()) {
diff --git a/tests/test_text.py b/tests/test_text.py
index d839420f4..53ccb3073 100644
--- a/tests/test_text.py
+++ b/tests/test_text.py
@@ -56,7 +56,7 @@ def test_read_text():
             assert entries[k].numpy().decode() + "\n" == v.decode()
 
 
-@pytest.mark.skip(reason="TODO")
+@pytest.mark.xfail(reason="TODO")
 def test_text_output_sequence():
     """Test case based on fashion mnist tutorial"""
     fashion_mnist = tf.keras.datasets.fashion_mnist
diff --git a/third_party/arrow.BUILD b/third_party/arrow.BUILD
index 004e716f1..85a5a5969 100644
--- a/third_party/arrow.BUILD
+++ b/third_party/arrow.BUILD
@@ -120,7 +120,6 @@ cc_library(
         "@rapidjson",
         "@snappy",
         "@thrift",
-        "@xsimd",
         "@zlib",
         "@zstd",
     ],
diff --git a/third_party/xsimd.BUILD b/third_party/xsimd.BUILD
deleted file mode 100644
index 8103e2c58..000000000
--- a/third_party/xsimd.BUILD
+++ /dev/null
@@ -1,31 +0,0 @@
-package(default_visibility = ["//visibility:public"])
-
-licenses(["notice"])  # BSD 3-Clause
-
-exports_files(["LICENSE"])
-
-cc_library(
-    name = "xsimd",
-    srcs = [],
-    hdrs = glob(
-        [
-            "include/xsimd/*.hpp",
-            "include/xsimd/config/*.hpp",
-            "include/xsimd/math/*.hpp",
-            "include/xsimd/memory/*.hpp",
-            "include/xsimd/stl/*.hpp",
-            "include/xsimd/types/*.hpp",
-        ],
-        exclude = [
-        ],
-    ),
-    copts = [],
-    defines = [],
-    includes = [
-        "include",
-    ],
-    linkopts = [],
-    visibility = ["//visibility:public"],
-    deps = [
-    ],
-)