From de737fc25ffe7eb6daf18341138a4e39b0304b69 Mon Sep 17 00:00:00 2001 From: Felipe Sere Date: Wed, 6 Jul 2011 10:46:28 +0200 Subject: [PATCH 1/8] Make the EmbeddedServerHelper more dynamic by allowing more parameters in the constructor --- .../testutils/EmbeddedServerHelper.java | 213 +++++++++- .../testutils/cassandra.yaml.template | 377 ++++++++++++++++++ 2 files changed, 572 insertions(+), 18 deletions(-) create mode 100644 core/src/main/resources/me/prettyprint/cassandra/testutils/cassandra.yaml.template diff --git a/core/src/main/java/me/prettyprint/cassandra/testutils/EmbeddedServerHelper.java b/core/src/main/java/me/prettyprint/cassandra/testutils/EmbeddedServerHelper.java index c38acfc1e..b7775e9cb 100644 --- a/core/src/main/java/me/prettyprint/cassandra/testutils/EmbeddedServerHelper.java +++ b/core/src/main/java/me/prettyprint/cassandra/testutils/EmbeddedServerHelper.java @@ -4,10 +4,25 @@ import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; +import java.io.FileInputStream; +import java.io.InputStreamReader; import java.io.OutputStream; +import java.io.BufferedReader; +import java.io.PrintWriter; +import java.io.FileWriter; +import java.io.FileReader; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; +import java.util.List; +import java.util.ArrayList; +import java.util.LinkedList; +import java.util.Collections; +import java.util.Map; +import java.util.HashMap; +import java.net.URL; + +import org.apache.commons.lang.text.StrSubstitutor; import org.apache.cassandra.config.CFMetaData; import org.apache.cassandra.config.ConfigurationException; @@ -25,23 +40,39 @@ /** * * @author Ran Tavory (rantav@gmail.com) - * + * @author Felipe Seré (felipesere@gmail.com) */ + public class EmbeddedServerHelper { private static Logger log = LoggerFactory.getLogger(EmbeddedServerHelper.class); - private static final String TMP = "tmp"; - private EmbeddedCassandraService cassandra; - private final String yamlFile; + private String log4jFile = null; + private final String clusterName; + private final String folder; + private final int port; + private boolean useDefaultLog4j; + static CassandraDaemon cassandraDaemon; + public EmbeddedServerHelper(String log4jFile, String clusterName, String folder, int port) { + this.log4jFile = log4jFile; + this.clusterName = clusterName; + this.folder = folder; + this.port = port; + this.useDefaultLog4j = false; + + } + public EmbeddedServerHelper() { - this("/cassandra.yaml"); + this("/log4j.properties", "Test Cluster", "tmp",9170); + + this.useDefaultLog4j = true; } - + + // Kept this constructor to be compatible public EmbeddedServerHelper(String yamlFile) { - this.yamlFile = yamlFile; + this(); } static ExecutorService executor = Executors.newSingleThreadExecutor(); @@ -55,18 +86,33 @@ public EmbeddedServerHelper(String yamlFile) { */ public void setup() throws TTransportException, IOException, InterruptedException, ConfigurationException { - // delete tmp dir first - rmdir(TMP); - // make a tmp dir and copy cassandra.yaml and log4j.properties to it - copy("/log4j.properties", TMP); - copy(yamlFile, TMP); - System.setProperty("cassandra.config", "file:" + TMP + yamlFile); - System.setProperty("log4j.configuration", "file:" + TMP + "/log4j.properties"); + + rmdir(folder); + + // Different copy strategies if he use the default log4j in the packge + // or one provided with a URL String + if( useDefaultLog4j) { + copy(log4jFile, folder); + } + else { + copyFile(log4jFile, folder); + } + // retrieve the cassandra.yaml.template from the JAR + URL url = EmbeddedServerHelper.class.getResource("cassandra.yaml.template"); + + if(url == null) { + throw new RuntimeException("Did not find the cassandra.yaml.template"); + } + + // Create a new file in the new folder without .template containing the following settings: + File cassandraConfig = createConfigFromTemplate(new File(url.getFile()), clusterName, folder, port); + + System.setProperty("cassandra.config", "file:" + folder + "/cassandra.yaml"); + System.setProperty("log4j.configuration", "file:" + folder + "/" + log4jFile.substring(log4jFile.lastIndexOf("/") +1)); System.setProperty("cassandra-foreground","true"); cleanupAndLeaveDirs(); loadSchemaFromYaml(); - //loadYamlTables(); log.info("Starting executor"); executor.execute(new CassandraRunner()); @@ -99,6 +145,34 @@ private static void rmdir(String dir) throws IOException { } } +/** + * Copies a resource from within the jar to a directory. + * + * @param resource + * @param directory + * @throws IOException + */ + private static void copy(String resource, String directory) throws IOException { + mkdir(directory); + InputStream is = EmbeddedServerHelper.class.getResourceAsStream(resource); + + if( is == null) { + throw new RuntimeException("Did not find resource: " + resource); + } + + String fileName = resource.substring(resource.lastIndexOf("/") + 1); + File file = new File(directory + System.getProperty("file.separator") + + fileName); + OutputStream out = new FileOutputStream(file); + byte buf[] = new byte[1024]; + int len; + while ((len = is.read(buf)) > 0) { + out.write(buf, 0, len); + } + out.close(); + is.close(); + } + /** * Copies a resource from within the jar to a directory. * @@ -106,22 +180,35 @@ private static void rmdir(String dir) throws IOException { * @param directory * @throws IOException */ - private static void copy(String resource, String directory) + private static void copyFile(String resource, String directory) throws IOException { mkdir(directory); - InputStream is = EmbeddedServerHelper.class.getResourceAsStream(resource); + String fileName = resource.substring(resource.lastIndexOf("/") + 1); + File resourceFile = new File(resource); + if (! resourceFile.exists() ) { + throw new RuntimeException("CopyFile did not find "+ resource); + } + + InputStream is = new FileInputStream(resourceFile); + File file = new File(directory + System.getProperty("file.separator") + fileName); OutputStream out = new FileOutputStream(file); byte buf[] = new byte[1024]; int len; + + if(is == null) { + throw new RuntimeException("Did not find " + resource ); + } + while ((len = is.read(buf)) > 0) { out.write(buf, 0, len); } out.close(); is.close(); } + /** * Creates a directory @@ -176,15 +263,105 @@ public static void mkdirs() } } + private static List readLinesOfResource(String filepath) { + + List result = new ArrayList(); + + BufferedReader br = null; + + + + String filename = filepath.substring(filepath.lastIndexOf("/")+1); + + try { + br = new BufferedReader(new InputStreamReader(EmbeddedServerHelper.class.getResourceAsStream(filename))); + + String line = null; + while((line = br.readLine()) != null ) { + result.add(line); + } + + } + catch (Exception e) { + log.info("Exception when reading file: " + e.getMessage()); + } + + if(result.size() == 0 ) { + throw new RuntimeException("Didn't read any lines? file: " + filepath); + } + + return result; + } + + private static File writeLines(String filename, List lines) { + + if(lines.size() == 0 ) { + throw new RuntimeException("Instructed to write 0 lines into file"); + } + + File file = new File(filename); + + try { + if (file.exists() ) { + file.delete(); + } + file.createNewFile(); + + PrintWriter pw = new PrintWriter(new FileWriter(file)); + + for(String line : lines) { + pw.println(line); + } + + }catch (Exception e) { + throw new RuntimeException("Exception when writing file: " + e.getMessage() + " " + file.getAbsolutePath()); + + } + + return file; + } + + public static void loadSchemaFromYaml() { EmbeddedSchemaLoader.loadSchema(); } + + private static File createConfigFromTemplate(File configTemplate, String clusterName, String directory, int port) { + + final String configFileName = "cassandra.yaml"; + + Map values = new HashMap(); + values.put("cluster_name", clusterName); + values.put("dir", directory); + values.put("port", port); + + StrSubstitutor sub = new StrSubstitutor(values); + + List lines = readLinesOfResource(configTemplate.getAbsolutePath()); + List result = new ArrayList(); + + if(lines == null) { + throw new RuntimeException("Could not read lines from template"); + } + + for(String line : lines) { + result.add(sub.replace(line)); + } + + File file = writeLines(directory+"/cassandra.yaml", result); + + if(file == null) { + throw new RuntimeException("Error writing new cassanda.yaml"); + } + + return file; + } class CassandraRunner implements Runnable { - + @Override public void run() { diff --git a/core/src/main/resources/me/prettyprint/cassandra/testutils/cassandra.yaml.template b/core/src/main/resources/me/prettyprint/cassandra/testutils/cassandra.yaml.template new file mode 100644 index 000000000..cf44d9732 --- /dev/null +++ b/core/src/main/resources/me/prettyprint/cassandra/testutils/cassandra.yaml.template @@ -0,0 +1,377 @@ +# Cassandra storage config YAML + +# NOTE: +# See http://wiki.apache.org/cassandra/StorageConfiguration for +# full explanations of configuration directives +# /NOTE + +# The name of the cluster. This is mainly used to prevent machines in +# one logical cluster from joining another. +cluster_name: '${cluster_name}' + +# You should always specify InitialToken when setting up a production +# cluster for the first time, and often when adding capacity later. +# The principle is that each node should be given an equal slice of +# the token ring; see http://wiki.apache.org/cassandra/Operations +# for more details. +# +# If blank, Cassandra will request a token bisecting the range of +# the heaviest-loaded existing node. If there is no load information +# available, such as is the case with a new cluster, it will pick +# a random token, which will lead to hot spots. +initial_token: + +# Set to true to make new [non-seed] nodes automatically migrate data +# to themselves from the pre-existing nodes in the cluster. Defaults +# to false because you can only bootstrap N machines at a time from +# an existing cluster of N, so if you are bringing up a cluster of +# 10 machines with 3 seeds you would have to do it in stages. Leaving +# this off for the initial start simplifies that. +auto_bootstrap: false + +# See http://wiki.apache.org/cassandra/HintedHandoff +hinted_handoff_enabled: true +# this defines the maximum amount of time a dead host will have hints +# generated. After it has been dead this long, hints will be dropped. +max_hint_window_in_ms: 3600000 # one hour +# Sleep this long after delivering each row or row fragment +hinted_handoff_throttle_delay_in_ms: 50 + +# authentication backend, implementing IAuthenticator; used to identify users +authenticator: org.apache.cassandra.auth.AllowAllAuthenticator + +# authorization backend, implementing IAuthority; used to limit access/provide permissions +authority: org.apache.cassandra.auth.AllowAllAuthority + +# The partitioner is responsible for distributing rows (by key) across +# nodes in the cluster. Any IPartitioner may be used, including your +# own as long as it is on the classpath. Out of the box, Cassandra +# provides org.apache.cassandra.dht.RandomPartitioner +# org.apache.cassandra.dht.ByteOrderedPartitioner, +# org.apache.cassandra.dht.OrderPreservingPartitioner (deprecated), +# and org.apache.cassandra.dht.CollatingOrderPreservingPartitioner +# (deprecated). +# +# - RandomPartitioner distributes rows across the cluster evenly by md5. +# When in doubt, this is the best option. +# - ByteOrderedPartitioner orders rows lexically by key bytes. BOP allows +# scanning rows in key order, but the ordering can generate hot spots +# for sequential insertion workloads. +# - OrderPreservingPartitioner is an obsolete form of BOP, that stores +# - keys in a less-efficient format and only works with keys that are +# UTF8-encoded Strings. +# - CollatingOPP colates according to EN,US rules rather than lexical byte +# ordering. Use this as an example if you need custom collation. +# +# See http://wiki.apache.org/cassandra/Operations for more on +# partitioners and token selection. +partitioner: org.apache.cassandra.dht.RandomPartitioner + +# directories where Cassandra should store data on disk. +data_file_directories: + - ${dir}/data + +# commit log +commitlog_directory: ${dir}/commitlog + +# saved caches +saved_caches_directory: ${dir}/saved_caches + +# Size to allow commitlog to grow to before creating a new segment +commitlog_rotation_threshold_in_mb: 128 + +# commitlog_sync may be either "periodic" or "batch." +# When in batch mode, Cassandra won't ack writes until the commit log +# has been fsynced to disk. It will wait up to +# CommitLogSyncBatchWindowInMS milliseconds for other writes, before +# performing the sync. +commitlog_sync: periodic + +# the other option is "periodic" where writes may be acked immediately +# and the CommitLog is simply synced every commitlog_sync_period_in_ms +# milliseconds. +commitlog_sync_period_in_ms: 10000 + +# any class that implements the SeedProvider interface and has a constructor that takes a Map of +# parameters will do. +seed_provider: + # Addresses of hosts that are deemed contact points. + # Cassandra nodes use this list of hosts to find each other and learn + # the topology of the ring. You must change this if you are running + # multiple nodes! + - class_name: org.apache.cassandra.locator.SimpleSeedProvider + parameters: + # seeds is actually a comma-delimited list of addresses. + - seeds: "127.0.0.1" + +# emergency pressure valve: each time heap usage after a full (CMS) +# garbage collection is above this fraction of the max, Cassandra will +# flush the largest memtables. +# +# Set to 1.0 to disable. Setting this lower than +# CMSInitiatingOccupancyFraction is not likely to be useful. +# +# RELYING ON THIS AS YOUR PRIMARY TUNING MECHANISM WILL WORK POORLY: +# it is most effective under light to moderate load, or read-heavy +# workloads; under truly massive write load, it will often be too +# little, too late. +flush_largest_memtables_at: 0.75 + +# emergency pressure valve #2: the first time heap usage after a full +# (CMS) garbage collection is above this fraction of the max, +# Cassandra will reduce cache maximum _capacity_ to the given fraction +# of the current _size_. Should usually be set substantially above +# flush_largest_memtables_at, since that will have less long-term +# impact on the system. +# +# Set to 1.0 to disable. Setting this lower than +# CMSInitiatingOccupancyFraction is not likely to be useful. +reduce_cache_sizes_at: 0.85 +reduce_cache_capacity_to: 0.6 + +# For workloads with more data than can fit in memory, Cassandra's +# bottleneck will be reads that need to fetch data from +# disk. "concurrent_reads" should be set to (16 * number_of_drives) in +# order to allow the operations to enqueue low enough in the stack +# that the OS and drives can reorder them. +# +# On the other hand, since writes are almost never IO bound, the ideal +# number of "concurrent_writes" is dependent on the number of cores in +# your system; (8 * number_of_cores) is a good rule of thumb. +concurrent_reads: 32 +concurrent_writes: 32 + +# Total memory to use for memtables. Cassandra will flush the largest +# memtable when this much memory is used. Prefer using this to +# the older, per-ColumnFamily memtable flush thresholds. +# If omitted, Cassandra will set it to 1/3 of the heap. +# If set to 0, only the old flush thresholds are used. +# memtable_total_space_in_mb: 2048 + +# This sets the amount of memtable flush writer threads. These will +# be blocked by disk io, and each one will hold a memtable in memory +# while blocked. If you have a large heap and many data directories, +# you can increase this value for better flush performance. +# By default this will be set to the amount of data directories defined. +#memtable_flush_writers: 1 + +# the number of full memtables to allow pending flush, that is, +# waiting for a writer thread. At a minimum, this should be set to +# the maximum number of secondary indexes created on a single CF. +memtable_flush_queue_size: 4 + +# Buffer size to use when performing contiguous column slices. +# Increase this to the size of the column slices you typically perform +sliced_buffer_size_in_kb: 64 + +# TCP port, for commands and data +storage_port: 7000 + +# Address to bind to and tell other Cassandra nodes to connect to. You +# _must_ change this if you want multiple nodes to be able to +# communicate! +# +# Leaving it blank leaves it up to InetAddress.getLocalHost(). This +# will always do the Right Thing *if* the node is properly configured +# (hostname, name resolution, etc), and the Right Thing is to use the +# address associated with the hostname (it might not be). +# +# Setting this to 0.0.0.0 is always wrong. +listen_address: localhost + +# The address to bind the Thrift RPC service to -- clients connect +# here. Unlike ListenAddress above, you *can* specify 0.0.0.0 here if +# you want Thrift to listen on all interfaces. +# +# Leaving this blank has the same effect it does for ListenAddress, +# (i.e. it will be based on the configured hostname of the node). +rpc_address: localhost +# port for Thrift to listen for clients on +rpc_port: ${port} + +# enable or disable keepalive on rpc connections +rpc_keepalive: true + +# Cassandra uses thread-per-client for client RPC. This can +# be expensive in memory used for thread stack for a large +# enough number of clients. (Hence, connection pooling is +# very, very strongly recommended.) +# +# Uncomment rpc_min|max|thread to set request pool size. +# You would primarily set max as a safeguard against misbehaved +# clients; if you do hit the max, Cassandra will block until +# one disconnects before accepting more. The defaults are +# min of 16 and max unlimited. +# +# rpc_min_threads: 16 +# rpc_max_threads: 2048 + +# uncomment to set socket buffer sizes on rpc connections +# rpc_send_buff_size_in_bytes: +# rpc_recv_buff_size_in_bytes: + +# Frame size for thrift (maximum field length). +# 0 disables TFramedTransport in favor of TSocket. This option +# is deprecated; we strongly recommend using Framed mode. +thrift_framed_transport_size_in_mb: 15 + +# The max length of a thrift message, including all fields and +# internal thrift overhead. +thrift_max_message_length_in_mb: 16 + +# Set to true to have Cassandra create a hard link to each sstable +# flushed or streamed locally in a backups/ subdirectory of the +# Keyspace data. Removing these links is the operator's +# responsibility. +incremental_backups: false + +# Whether or not to take a snapshot before each compaction. Be +# careful using this option, since Cassandra won't clean up the +# snapshots for you. Mostly useful if you're paranoid when there +# is a data format change. +snapshot_before_compaction: false + +# change this to increase the compaction thread's priority. In java, 1 is the +# lowest priority and that is our default. +# compaction_thread_priority: 1 + +# Add column indexes to a row after its contents reach this size. +# Increase if your column values are large, or if you have a very large +# number of columns. The competing causes are, Cassandra has to +# deserialize this much of the row to read a single column, so you want +# it to be small - at least if you do many partial-row reads - but all +# the index data is read for each access, so you don't want to generate +# that wastefully either. +column_index_size_in_kb: 64 + +# Size limit for rows being compacted in memory. Larger rows will spill +# over to disk and use a slower two-pass compaction process. A message +# will be logged specifying the row key. +in_memory_compaction_limit_in_mb: 64 + +# Number of compaction threads. This default to the number of processors, +# enabling multiple compactions to execute at once. Using more than one +# thread is highly recommended to preserve read performance in a mixed +# read/write workload as this avoids sstables from accumulating during long +# running compactions. The default is usually fine and if you experience +# problems with compaction running too slowly or too fast, you should look at +# compaction_throughput_mb_per_sec first. +# Uncomment to make compaction mono-threaded. +#concurrent_compactors: 1 + +# Throttles compaction to the given total throughput across the entire +# system. The faster you insert data, the faster you need to compact in +# order to keep the sstable count down, but in general, setting this to +# 16 to 32 times the rate you are inserting data is more than sufficient. +# Setting this to 0 disables throttling. +compaction_throughput_mb_per_sec: 16 + +# Track cached row keys during compaction, and re-cache their new +# positions in the compacted sstable. Disable if you use really large +# key caches. +compaction_preheat_key_cache: true + +# Time to wait for a reply from other nodes before failing the command +rpc_timeout_in_ms: 10000 + +# phi value that must be reached for a host to be marked down. +# most users should never need to adjust this. +# phi_convict_threshold: 8 + +# endpoint_snitch -- Set this to a class that implements +# IEndpointSnitch, which will let Cassandra know enough +# about your network topology to route requests efficiently. +# Out of the box, Cassandra provides +# - org.apache.cassandra.locator.SimpleSnitch: +# Treats Strategy order as proximity. This improves cache locality +# when disabling read repair, which can further improve throughput. +# - org.apache.cassandra.locator.RackInferringSnitch: +# Proximity is determined by rack and data center, which are +# assumed to correspond to the 3rd and 2nd octet of each node's +# IP address, respectively +# org.apache.cassandra.locator.PropertyFileSnitch: +# - Proximity is determined by rack and data center, which are +# explicitly configured in cassandra-topology.properties. +endpoint_snitch: org.apache.cassandra.locator.SimpleSnitch + +# dynamic_snitch -- This boolean controls whether the above snitch is +# wrapped with a dynamic snitch, which will monitor read latencies +# and avoid reading from hosts that have slowed (due to compaction, +# for instance) +dynamic_snitch: true +# controls how often to perform the more expensive part of host score +# calculation +dynamic_snitch_update_interval_in_ms: 100 +# controls how often to reset all host scores, allowing a bad host to +# possibly recover +dynamic_snitch_reset_interval_in_ms: 600000 +# if set greater than zero and read_repair_chance is < 1.0, this will allow +# 'pinning' of replicas to hosts in order to increase cache capacity. +# The badness threshold will control how much worse the pinned host has to be +# before the dynamic snitch will prefer other replicas over it. This is +# expressed as a double which represents a percentage. Thus, a value of +# 0.2 means Cassandra would continue to prefer the static snitch values +# until the pinned host was 20% worse than the fastest. +dynamic_snitch_badness_threshold: 0.0 + +# request_scheduler -- Set this to a class that implements +# RequestScheduler, which will schedule incoming client requests +# according to the specific policy. This is useful for multi-tenancy +# with a single Cassandra cluster. +# NOTE: This is specifically for requests from the client and does +# not affect inter node communication. +# org.apache.cassandra.scheduler.NoScheduler - No scheduling takes place +# org.apache.cassandra.scheduler.RoundRobinScheduler - Round robin of +# client requests to a node with a separate queue for each +# request_scheduler_id. The scheduler is further customized by +# request_scheduler_options as described below. +request_scheduler: org.apache.cassandra.scheduler.NoScheduler + +# Scheduler Options vary based on the type of scheduler +# NoScheduler - Has no options +# RoundRobin +# - throttle_limit -- The throttle_limit is the number of in-flight +# requests per client. Requests beyond +# that limit are queued up until +# running requests can complete. +# The value of 80 here is twice the number of +# concurrent_reads + concurrent_writes. +# - default_weight -- default_weight is optional and allows for +# overriding the default which is 1. +# - weights -- Weights are optional and will default to 1 or the +# overridden default_weight. The weight translates into how +# many requests are handled during each turn of the +# RoundRobin, based on the scheduler id. +# +# request_scheduler_options: +# throttle_limit: 80 +# default_weight: 5 +# weights: +# Keyspace1: 1 +# Keyspace2: 5 + +# request_scheduler_id -- An identifer based on which to perform +# the request scheduling. Currently the only valid option is keyspace. +# request_scheduler_id: keyspace + +# The Index Interval determines how large the sampling of row keys +# is for a given SSTable. The larger the sampling, the more effective +# the index is at the cost of space. +index_interval: 128 + +# Enable or disable inter-node encryption +# Default settings are TLS v1, RSA 1024-bit keys (it is imperative that +# users generate their own keys) TLS_RSA_WITH_AES_128_CBC_SHA as the cipher +# suite for authentication, key exchange and encryption of the actual data transfers. +# NOTE: No custom encryption options are enabled at the moment +# The available internode options are : all, none +# +# The passwords used in these options must match the passwords used when generating +# the keystore and truststore. For instructions on generating these files, see: +# http://download.oracle.com/javase/6/docs/technotes/guides/security/jsse/JSSERefGuide.html#CreateKeystore +encryption_options: + internode_encryption: none + keystore: conf/.keystore + keystore_password: cassandra + truststore: conf/.truststore + truststore_password: cassandra From c2524cff02ed8f39a05f0a293c1c71fc0ee7af70 Mon Sep 17 00:00:00 2001 From: Felipe Sere Date: Mon, 22 Aug 2011 11:11:25 +0200 Subject: [PATCH 2/8] fix: use the proper platform path separator --- .../cassandra/testutils/EmbeddedServerHelper.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/me/prettyprint/cassandra/testutils/EmbeddedServerHelper.java b/core/src/main/java/me/prettyprint/cassandra/testutils/EmbeddedServerHelper.java index b7775e9cb..4dd3e2faf 100644 --- a/core/src/main/java/me/prettyprint/cassandra/testutils/EmbeddedServerHelper.java +++ b/core/src/main/java/me/prettyprint/cassandra/testutils/EmbeddedServerHelper.java @@ -333,6 +333,8 @@ private static File createConfigFromTemplate(File configTemplate, String cluster final String configFileName = "cassandra.yaml"; Map values = new HashMap(); + final String pathSeparator = File.separator; + values.put("cluster_name", clusterName); values.put("dir", directory); values.put("port", port); @@ -350,10 +352,10 @@ private static File createConfigFromTemplate(File configTemplate, String cluster result.add(sub.replace(line)); } - File file = writeLines(directory+"/cassandra.yaml", result); + File file = writeLines(directory+pathSeparator+configFileName, result); if(file == null) { - throw new RuntimeException("Error writing new cassanda.yaml"); + throw new RuntimeException("Error writing new " + configFileName); } return file; From 5702cfad615784f521ad84d5dbf95167f25a65c8 Mon Sep 17 00:00:00 2001 From: Felipe Sere Date: Wed, 24 Aug 2011 12:00:49 +0200 Subject: [PATCH 3/8] Add an object to configure the EmbeddedServerHelper --- .../testutils/EmbeddedServerConfigurator.java | 110 ++++++++++++++++++ 1 file changed, 110 insertions(+) create mode 100644 core/src/main/java/me/prettyprint/cassandra/testutils/EmbeddedServerConfigurator.java diff --git a/core/src/main/java/me/prettyprint/cassandra/testutils/EmbeddedServerConfigurator.java b/core/src/main/java/me/prettyprint/cassandra/testutils/EmbeddedServerConfigurator.java new file mode 100644 index 000000000..2ed874a44 --- /dev/null +++ b/core/src/main/java/me/prettyprint/cassandra/testutils/EmbeddedServerConfigurator.java @@ -0,0 +1,110 @@ +package me.prettyprint.cassandra.testutils; + +import me.prettyprint.cassandra.service.template.MappedColumnFamilyResult; +import org.yaml.snakeyaml.Yaml; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * @author felipesere@gmail.com + */ + +public class EmbeddedServerConfigurator { + private String clusterName; + private String folder; + private int thriftPort; + + private String yamlPath; + private String log4jPath; + private boolean usePackagedLog4j; + + public EmbeddedServerConfigurator() { + this.setClusterName("Test Cluster"); + this.setFolder("tmp"); + this.setThriftPort(9170); + this.setLog4jPath("log4j.properties"); + this.setUsePackagedLog4j(true); + } + + public static EmbeddedServerConfigurator createFromYaml(String path) { + EmbeddedServerConfigurator result = new EmbeddedServerConfigurator(); + if(path != null) { + // We need to process the yaml file to fill in the missing variables + Yaml yaml = new Yaml(); + try { + FileInputStream fis = new FileInputStream(path); + Map map = (Map) yaml.load(fis); + + Set keys = map.keySet(); + + result.setClusterName((String) map.get("cluster_name")); + result.setThriftPort((Integer) map.get("rpc_port")); + List dirs = (List) map.get("data_file_directories"); + + result.setFolder(dirs.get(0).substring(0,dirs.get(0).indexOf(File.separator))); + + } + catch( FileNotFoundException fnf) { + throw new RuntimeException("File not found: " + fnf.getMessage()); + } + + } + + return result; + } + + public String getClusterName() { + return clusterName; + } + + public void setClusterName(String clusterName) { + this.clusterName = clusterName; + } + + public String getFolder() { + return folder; + } + + public void setFolder(String folder) { + this.folder = folder; + } + + public int getThriftPort() { + return thriftPort; + } + + public void setThriftPort(int thriftPort) { + this.thriftPort = thriftPort; + } + + public String getYamlPath() { + return yamlPath; + } + + public void setYamlPath(String yamlPath) { + + } + + public String getLog4jPath() { + return log4jPath; + } + + public void setLog4jPath(String log4jPath) { + this.log4jPath = log4jPath; + } + + public boolean isUsePackagedLog4j() { + return usePackagedLog4j; + } + + public void setUsePackagedLog4j(boolean usePackagedLog4j) { + this.usePackagedLog4j = usePackagedLog4j; + } +} + + From 7b538bdac7127571e3703628b2f0ac4e511ad7f1 Mon Sep 17 00:00:00 2001 From: Felipe Sere Date: Wed, 24 Aug 2011 12:03:31 +0200 Subject: [PATCH 4/8] Upgrade the version of snakeyaml to 1.9 and change its scope --- core/pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/pom.xml b/core/pom.xml index a0bdf7a1f..68f0bb704 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -117,8 +117,8 @@ org.yaml snakeyaml - 1.6 - test + 1.9 + compile com.google.guava From 6f5553667efc7464751318097e74c8e40c3e2e74 Mon Sep 17 00:00:00 2001 From: Felipe Sere Date: Wed, 24 Aug 2011 12:06:13 +0200 Subject: [PATCH 5/8] Use the config object and simplify methods --- .../testutils/EmbeddedServerHelper.java | 268 +++++++----------- 1 file changed, 103 insertions(+), 165 deletions(-) diff --git a/core/src/main/java/me/prettyprint/cassandra/testutils/EmbeddedServerHelper.java b/core/src/main/java/me/prettyprint/cassandra/testutils/EmbeddedServerHelper.java index 4dd3e2faf..c4bf24fad 100644 --- a/core/src/main/java/me/prettyprint/cassandra/testutils/EmbeddedServerHelper.java +++ b/core/src/main/java/me/prettyprint/cassandra/testutils/EmbeddedServerHelper.java @@ -1,37 +1,21 @@ package me.prettyprint.cassandra.testutils; -import java.io.File; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.FileInputStream; -import java.io.InputStreamReader; -import java.io.OutputStream; -import java.io.BufferedReader; -import java.io.PrintWriter; -import java.io.FileWriter; -import java.io.FileReader; +import java.io.*; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; import java.util.List; import java.util.ArrayList; -import java.util.LinkedList; -import java.util.Collections; import java.util.Map; import java.util.HashMap; import java.net.URL; import org.apache.commons.lang.text.StrSubstitutor; -import org.apache.cassandra.config.CFMetaData; import org.apache.cassandra.config.ConfigurationException; import org.apache.cassandra.config.DatabaseDescriptor; -import org.apache.cassandra.config.KSMetaData; -import org.apache.cassandra.contrib.utils.service.CassandraServiceDataCleaner; import org.apache.cassandra.db.commitlog.CommitLog; import org.apache.cassandra.io.util.FileUtils; -import org.apache.cassandra.service.EmbeddedCassandraService; import org.apache.cassandra.thrift.CassandraDaemon; import org.apache.thrift.transport.TTransportException; import org.slf4j.Logger; @@ -46,35 +30,24 @@ public class EmbeddedServerHelper { private static Logger log = LoggerFactory.getLogger(EmbeddedServerHelper.class); - private EmbeddedCassandraService cassandra; - private String log4jFile = null; - private final String clusterName; - private final String folder; - private final int port; - private boolean useDefaultLog4j; - - static CassandraDaemon cassandraDaemon; - - public EmbeddedServerHelper(String log4jFile, String clusterName, String folder, int port) { - this.log4jFile = log4jFile; - this.clusterName = clusterName; - this.folder = folder; - this.port = port; - this.useDefaultLog4j = false; - - } - + private static EmbeddedServerConfigurator configurator; + + CassandraDaemon cassandraDaemon; + public EmbeddedServerHelper() { - this("/log4j.properties", "Test Cluster", "tmp",9170); - - this.useDefaultLog4j = true; + this(new EmbeddedServerConfigurator()); + } + + // Main constructor that is called from all other constructors + public EmbeddedServerHelper(EmbeddedServerConfigurator cfg) { + this.configurator = cfg; } // Kept this constructor to be compatible public EmbeddedServerHelper(String yamlFile) { - this(); + this(EmbeddedServerConfigurator.createFromYaml(yamlFile)); } - + static ExecutorService executor = Executors.newSingleThreadExecutor(); /** @@ -86,35 +59,26 @@ public EmbeddedServerHelper(String yamlFile) { */ public void setup() throws TTransportException, IOException, InterruptedException, ConfigurationException { - - rmdir(folder); - - // Different copy strategies if he use the default log4j in the packge - // or one provided with a URL String - if( useDefaultLog4j) { - copy(log4jFile, folder); - } - else { - copyFile(log4jFile, folder); - } - // retrieve the cassandra.yaml.template from the JAR - URL url = EmbeddedServerHelper.class.getResource("cassandra.yaml.template"); - - if(url == null) { - throw new RuntimeException("Did not find the cassandra.yaml.template"); - } + + // Delete the old folder and create a new one + rmdir(configurator.getFolder()); + mkdir(configurator.getFolder()); + + // Copy the log4j file into the destination folder specified in the configurator + copy(configurator); - // Create a new file in the new folder without .template containing the following settings: - File cassandraConfig = createConfigFromTemplate(new File(url.getFile()), clusterName, folder, port); - - System.setProperty("cassandra.config", "file:" + folder + "/cassandra.yaml"); - System.setProperty("log4j.configuration", "file:" + folder + "/" + log4jFile.substring(log4jFile.lastIndexOf("/") +1)); + // Create a new cassandra config based on the information in the EmbeddedServerConfigurator + loadConfigIntoDestination(configurator); + + System.setProperty("cassandra.config", "file:" + configurator.getFolder() + File.separator + "cassandra.yaml"); + System.setProperty("log4j.configuration", "file:" + configurator.getFolder() + File.separator + "log4j.properties"); System.setProperty("cassandra-foreground","true"); cleanupAndLeaveDirs(); + loadSchemaFromYaml(); + log.info("Starting executor"); - executor.execute(new CassandraRunner()); log.info("Started executor"); try @@ -128,87 +92,73 @@ public void setup() throws TTransportException, IOException, } } - - - public static void teardown() { - //if ( cassandraDaemon != null ) - //cassandraDaemon.stop(); + public static void teardown() throws IOException { executor.shutdown(); executor.shutdownNow(); + + rmdir(configurator.getFolder()); log.info("Teardown complete"); } - private static void rmdir(String dir) throws IOException { + private static void rmdir(String dir) throws IOException{ + if(dir.contains(File.separator)) { + dir = dir.substring(0,dir.indexOf(File.separator)); + } + log.info("Deleting " + dir); File dirFile = new File(dir); if (dirFile.exists()) { - FileUtils.deleteRecursive(new File(dir)); - } - } + FileUtils.deleteRecursive(dirFile); -/** - * Copies a resource from within the jar to a directory. - * - * @param resource - * @param directory - * @throws IOException - */ - private static void copy(String resource, String directory) throws IOException { - mkdir(directory); - InputStream is = EmbeddedServerHelper.class.getResourceAsStream(resource); - - if( is == null) { - throw new RuntimeException("Did not find resource: " + resource); - } - - String fileName = resource.substring(resource.lastIndexOf("/") + 1); - File file = new File(directory + System.getProperty("file.separator") - + fileName); - OutputStream out = new FileOutputStream(file); - byte buf[] = new byte[1024]; - int len; - while ((len = is.read(buf)) > 0) { - out.write(buf, 0, len); } - out.close(); - is.close(); } - - /** - * Copies a resource from within the jar to a directory. - * - * @param resource - * @param directory - * @throws IOException - */ - private static void copyFile(String resource, String directory) - throws IOException { - mkdir(directory); - - String fileName = resource.substring(resource.lastIndexOf("/") + 1); - File resourceFile = new File(resource); - if (! resourceFile.exists() ) { - throw new RuntimeException("CopyFile did not find "+ resource); - } - - InputStream is = new FileInputStream(resourceFile); - - File file = new File(directory + System.getProperty("file.separator") - + fileName); - OutputStream out = new FileOutputStream(file); - byte buf[] = new byte[1024]; - int len; - - if(is == null) { - throw new RuntimeException("Did not find " + resource ); - } - - while ((len = is.read(buf)) > 0) { - out.write(buf, 0, len); + + /** + * Copy the log4j file into the the destination folder specified in configurator + * @param configurator Holds information where to copy the log4j and whether to use the packaged one + */ + private static void copy(EmbeddedServerConfigurator configurator) { + String dest = configurator.getFolder() + File.separator + "log4j.properties"; + InputStream is = null; + if(configurator.isUsePackagedLog4j()) { + is = EmbeddedServerHelper.class.getResourceAsStream(File.separator + "log4j.properties"); + } + else { + try { + is = new FileInputStream(configurator.getLog4jPath()); + } + catch (FileNotFoundException fnfe) { + is = EmbeddedServerHelper.class.getResourceAsStream(File.separator + "log4j.properties"); + } + } + + File f = new File(dest); + OutputStream out; + try { + out = new FileOutputStream(f); + } + catch( FileNotFoundException fnfe ){ + try { + f.createNewFile(); + out = new FileOutputStream(f); + }catch (IOException ioe) { + throw new RuntimeException("Caught an IOException " + ioe.getLocalizedMessage()); + } + } + + byte buf[] = new byte[1024]; + int len; + + try { + while ((len = is.read(buf)) > 0) { + out.write(buf, 0, len); + } + out.close(); + is.close(); + } + catch (IOException ioe) { + throw new RuntimeException("Caught an IOException " + ioe.getLocalizedMessage()); + } } - out.close(); - is.close(); - } - /** * Creates a directory @@ -218,6 +168,7 @@ private static void copyFile(String resource, String directory) */ private static void mkdir(String dir) throws IOException { FileUtils.createDirectory(dir); + log.info("Creating directory: " + dir); } @@ -267,16 +218,12 @@ private static List readLinesOfResource(String filepath) { List result = new ArrayList(); - BufferedReader br = null; - - - - String filename = filepath.substring(filepath.lastIndexOf("/")+1); + BufferedReader br; try { - br = new BufferedReader(new InputStreamReader(EmbeddedServerHelper.class.getResourceAsStream(filename))); + br = new BufferedReader(new InputStreamReader(EmbeddedServerHelper.class.getResourceAsStream(filepath), "UTF-8")); - String line = null; + String line; while((line = br.readLine()) != null ) { result.add(line); } @@ -286,10 +233,6 @@ private static List readLinesOfResource(String filepath) { log.info("Exception when reading file: " + e.getMessage()); } - if(result.size() == 0 ) { - throw new RuntimeException("Didn't read any lines? file: " + filepath); - } - return result; } @@ -325,40 +268,35 @@ private static File writeLines(String filename, List lines) { public static void loadSchemaFromYaml() { EmbeddedSchemaLoader.loadSchema(); - } - private static File createConfigFromTemplate(File configTemplate, String clusterName, String directory, int port) { + //private static File loadConfigIntoDestination(File configTemplate, String clusterName, String directory, int port) { + private static void loadConfigIntoDestination(EmbeddedServerConfigurator cfg) { + + URL url = EmbeddedServerHelper.class.getResource("cassandra.yaml.template"); + final String configFileName = "cassandra.yaml"; - - Map values = new HashMap(); final String pathSeparator = File.separator; - values.put("cluster_name", clusterName); - values.put("dir", directory); - values.put("port", port); + + Map values = new HashMap(); + values.put("cluster_name", cfg.getClusterName()); + values.put("dir", cfg.getFolder()); + values.put("port", Integer.toString(cfg.getThriftPort())); StrSubstitutor sub = new StrSubstitutor(values); - - List lines = readLinesOfResource(configTemplate.getAbsolutePath()); + + + List lines = readLinesOfResource(new File(url.getFile()).getName()); List result = new ArrayList(); - if(lines == null) { - throw new RuntimeException("Could not read lines from template"); - } - for(String line : lines) { result.add(sub.replace(line)); } - - File file = writeLines(directory+pathSeparator+configFileName, result); - - if(file == null) { - throw new RuntimeException("Error writing new " + configFileName); - } - - return file; + + + writeLines(cfg.getFolder()+pathSeparator+configFileName, result); } From d4f24e778899cfe6ba20c307992d943b6f270c3f Mon Sep 17 00:00:00 2001 From: Felipe Sere Date: Wed, 24 Aug 2011 12:07:10 +0200 Subject: [PATCH 6/8] Add tests for the constructors of EmbeddedServerHelper --- .../AdvancedConfiguratorConstructorTest.java | 62 +++ .../DefaultConstructorTest.java | 45 +++ ...ternalYamlConfiguratorConstructorTest.java | 85 ++++ .../SimpleConfiguratorConstructorTest.java | 34 ++ .../EmbeddedServerTests/test-cassandra.yaml | 374 ++++++++++++++++++ 5 files changed, 600 insertions(+) create mode 100644 core/src/test/java/me/prettyprint/cassandra/utils/EmbeddedServerTests/AdvancedConfiguratorConstructorTest.java create mode 100644 core/src/test/java/me/prettyprint/cassandra/utils/EmbeddedServerTests/DefaultConstructorTest.java create mode 100644 core/src/test/java/me/prettyprint/cassandra/utils/EmbeddedServerTests/ExternalYamlConfiguratorConstructorTest.java create mode 100644 core/src/test/java/me/prettyprint/cassandra/utils/EmbeddedServerTests/SimpleConfiguratorConstructorTest.java create mode 100644 core/src/test/resources/me/prettyprint/cassandra/utils/EmbeddedServerTests/test-cassandra.yaml diff --git a/core/src/test/java/me/prettyprint/cassandra/utils/EmbeddedServerTests/AdvancedConfiguratorConstructorTest.java b/core/src/test/java/me/prettyprint/cassandra/utils/EmbeddedServerTests/AdvancedConfiguratorConstructorTest.java new file mode 100644 index 000000000..bc0067568 --- /dev/null +++ b/core/src/test/java/me/prettyprint/cassandra/utils/EmbeddedServerTests/AdvancedConfiguratorConstructorTest.java @@ -0,0 +1,62 @@ +package me.prettyprint.cassandra.utils.EmbeddedServerTests; + +import me.prettyprint.cassandra.service.CassandraHostConfigurator; +import me.prettyprint.cassandra.service.ThriftCluster; +import me.prettyprint.cassandra.testutils.EmbeddedServerConfigurator; +import me.prettyprint.cassandra.testutils.EmbeddedServerHelper; +import org.apache.cassandra.thrift.Cassandra; +import org.apache.cassandra.utils.EstimatedHistogram; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.util.concurrent.TimeUnit; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +public class AdvancedConfiguratorConstructorTest { + static String clusterName = "Alfonso"; + static int thriftPort = 2204; + ThriftCluster cassandraCluster; + static EmbeddedServerHelper embedded; + + + @BeforeClass + public static void testSimpleConfiguratorConstructor() { + + EmbeddedServerConfigurator esc = new EmbeddedServerConfigurator(); + esc.setFolder("apple/pie"); + esc.setClusterName(clusterName); + esc.setThriftPort(thriftPort); + + embedded = new EmbeddedServerHelper(esc); + try { + embedded.setup(); + }catch (Exception e) {} + } + + @AfterClass + public static void teardown() { + try { + embedded.teardown(); + } + catch ( Exception e){} + } + + @Test + public void testConnection() { + CassandraHostConfigurator chc = new CassandraHostConfigurator(); + chc = new CassandraHostConfigurator("localhost:" + thriftPort); + + try { + cassandraCluster = new ThriftCluster("Test Cluster", chc); + assertTrue(cassandraCluster.describeClusterName().equals(clusterName)); + } + catch(Exception e) { + // Should never have reached here + assertTrue("Exception " + e.toString(),false); + } + + } +} diff --git a/core/src/test/java/me/prettyprint/cassandra/utils/EmbeddedServerTests/DefaultConstructorTest.java b/core/src/test/java/me/prettyprint/cassandra/utils/EmbeddedServerTests/DefaultConstructorTest.java new file mode 100644 index 000000000..c041ea9c5 --- /dev/null +++ b/core/src/test/java/me/prettyprint/cassandra/utils/EmbeddedServerTests/DefaultConstructorTest.java @@ -0,0 +1,45 @@ +package me.prettyprint.cassandra.utils.EmbeddedServerTests; + +import me.prettyprint.cassandra.testutils.EmbeddedSchemaLoader; +import me.prettyprint.cassandra.testutils.EmbeddedServerConfigurator; +import me.prettyprint.cassandra.testutils.EmbeddedServerHelper; + +import org.apache.cassandra.service.EmbeddedCassandraService; +import org.slf4j.LoggerFactory; +import org.slf4j.Logger; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import org.junit.Test; + +import java.util.concurrent.TimeUnit; + +/** + * @author Felipe Seré + */ +public class DefaultConstructorTest { + public static Logger log = LoggerFactory.getLogger(DefaultConstructorTest.class); + + @Test + public void testDefaultConstructor() { + + EmbeddedServerHelper embedded = new EmbeddedServerHelper(); + try { + embedded.setup(); + TimeUnit.SECONDS.sleep(10); + embedded.teardown(); + + } + catch(Exception e) { + // Should never have reached here + assertTrue("Exception " + e.toString(),false); + + } + + embedded = null; + + assertTrue(true); + } +} diff --git a/core/src/test/java/me/prettyprint/cassandra/utils/EmbeddedServerTests/ExternalYamlConfiguratorConstructorTest.java b/core/src/test/java/me/prettyprint/cassandra/utils/EmbeddedServerTests/ExternalYamlConfiguratorConstructorTest.java new file mode 100644 index 000000000..748d6c800 --- /dev/null +++ b/core/src/test/java/me/prettyprint/cassandra/utils/EmbeddedServerTests/ExternalYamlConfiguratorConstructorTest.java @@ -0,0 +1,85 @@ +package me.prettyprint.cassandra.utils.EmbeddedServerTests; + +import me.prettyprint.cassandra.service.CassandraHostConfigurator; +import me.prettyprint.cassandra.service.ThriftCluster; +import me.prettyprint.cassandra.testutils.EmbeddedServerConfigurator; +import me.prettyprint.cassandra.testutils.EmbeddedServerHelper; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import sun.misc.IOUtils; + +import java.awt.geom.Path2D; +import java.io.*; +import java.net.URL; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +public class ExternalYamlConfiguratorConstructorTest { + static String clusterName = "Alfonso"; + static int thriftPort = 2204; + ThriftCluster cassandraCluster; + static EmbeddedServerHelper embedded; + static File yamlFile; + private static final String sep = File.separator; + private static final String pathName = sep + "me" + sep + "prettyprint" + sep + "cassandra" + sep + "utils" + sep + "EmbeddedServerTests" + sep + "test-cassandra.yaml"; + + @BeforeClass + public static void getYamlFile() throws IOException { + + // Get the resource + URL resource = ExternalYamlConfiguratorConstructorTest.class.getResource(pathName); + yamlFile = File.createTempFile("test-cassandra","yaml"); + + + // Transfer it to a tempfile for later use + BufferedReader br = new BufferedReader(new InputStreamReader(ExternalYamlConfiguratorConstructorTest.class.getResourceAsStream(pathName))); + PrintWriter pw = new PrintWriter(yamlFile); + String line = null; + while((line = br.readLine()) != null) { + pw.println(line); + } + + br.close(); + pw.close(); + + } + + @Test + public void constructorTest() { + + EmbeddedServerConfigurator esc = EmbeddedServerConfigurator.createFromYaml(yamlFile.getPath()); + + assertTrue("ClusterName mismatch: esc " + esc.getClusterName() + " and " + clusterName, esc.getClusterName().equals(clusterName)); + assertTrue("Thirft port mismatch: esc " + esc.getThriftPort() + " and " + thriftPort , esc.getThriftPort() == thriftPort); + embedded = new EmbeddedServerHelper(esc); + try { + embedded.setup(); + }catch (Exception e) {} + } + + @AfterClass + public static void teardown() { + try { + embedded.teardown(); + } + catch ( Exception e){} + } + + @Test + public void testConnection() { + CassandraHostConfigurator chc = new CassandraHostConfigurator(); + chc = new CassandraHostConfigurator("localhost:" + thriftPort); + + try { + cassandraCluster = new ThriftCluster("Test Cluster", chc); + assertTrue(cassandraCluster.describeClusterName().equals(clusterName)); + } + catch(Exception e) { + // Should never have reached here + assertTrue("Exception " + e.toString(),false); + } + + } +} diff --git a/core/src/test/java/me/prettyprint/cassandra/utils/EmbeddedServerTests/SimpleConfiguratorConstructorTest.java b/core/src/test/java/me/prettyprint/cassandra/utils/EmbeddedServerTests/SimpleConfiguratorConstructorTest.java new file mode 100644 index 000000000..900cc31f9 --- /dev/null +++ b/core/src/test/java/me/prettyprint/cassandra/utils/EmbeddedServerTests/SimpleConfiguratorConstructorTest.java @@ -0,0 +1,34 @@ +package me.prettyprint.cassandra.utils.EmbeddedServerTests; + +import me.prettyprint.cassandra.testutils.EmbeddedServerConfigurator; +import me.prettyprint.cassandra.testutils.EmbeddedServerHelper; +import org.junit.Test; + +import java.util.concurrent.TimeUnit; + +import static org.junit.Assert.assertTrue; + +public class SimpleConfiguratorConstructorTest { + @Test + public void testSimpleConfiguratorConstructor() { + + EmbeddedServerConfigurator esc = new EmbeddedServerConfigurator(); + + EmbeddedServerHelper embedded = new EmbeddedServerHelper(esc); + try { + embedded.setup(); + TimeUnit.SECONDS.sleep(10); + embedded.teardown(); + + } + catch(Exception e) { + // Should never have reached here + assertTrue("Exception " + e.toString(),false); + + } + + embedded = null; + + assertTrue(true); + } +} diff --git a/core/src/test/resources/me/prettyprint/cassandra/utils/EmbeddedServerTests/test-cassandra.yaml b/core/src/test/resources/me/prettyprint/cassandra/utils/EmbeddedServerTests/test-cassandra.yaml new file mode 100644 index 000000000..576af20ce --- /dev/null +++ b/core/src/test/resources/me/prettyprint/cassandra/utils/EmbeddedServerTests/test-cassandra.yaml @@ -0,0 +1,374 @@ +# Cassandra storage config YAML + +#NOTE !!!!!!!! NOTE +# See http://wiki.apache.org/cassandra/StorageConfiguration for +# full explanations of configuration directives +#NOTE !!!!!!!! NOTE + +# The name of the cluster. This is mainly used to prevent machines in +# one logical cluster from joining another. +cluster_name: 'Alfonso' + +# You should always specify InitialToken when setting up a production +# cluster for the first time, and often when adding capacity later. +# The principle is that each node should be given an equal slice of +# the token ring; see http://wiki.apache.org/cassandra/Operations +# for more details. +# +# If blank, Cassandra will request a token bisecting the range of +# the heaviest-loaded existing node. If there is no load information +# available, such as is the case with a new cluster, it will pick +# a random token, which will lead to hot spots. +initial_token: + +# Set to true to make new [non-seed] nodes automatically migrate data +# to themselves from the pre-existing nodes in the cluster. Defaults +# to false because you can only bootstrap N machines at a time from +# an existing cluster of N, so if you are bringing up a cluster of +# 10 machines with 3 seeds you would have to do it in stages. Leaving +# this off for the initial start simplifies that. +auto_bootstrap: false + +# See http://wiki.apache.org/cassandra/HintedHandoff +hinted_handoff_enabled: true + +# authentication backend, implementing IAuthenticator; used to identify users +authenticator: org.apache.cassandra.auth.AllowAllAuthenticator + +# authorization backend, implementing IAuthority; used to limit access/provide permissions +authority: org.apache.cassandra.auth.AllowAllAuthority + +# any IPartitioner may be used, including your own as long as it is on +# the classpath. Out of the box, Cassandra provides +# org.apache.cassandra.dht.RandomPartitioner +# org.apache.cassandra.dht.ByteOrderedPartitioner, +# org.apache.cassandra.dht.OrderPreservingPartitioner, and +# org.apache.cassandra.dht.CollatingOrderPreservingPartitioner. +# (CollatingOPP colates according to EN,US rules, not naive byte +# ordering. Use this as an example if you need locale-aware collation.) +partitioner: org.apache.cassandra.dht.OrderPreservingPartitioner + +# directories where Cassandra should store data on disk. +data_file_directories: + - tmp/var/lib/cassandra/data + +# commit log +commitlog_directory: tmp/var/lib/cassandra/commitlog + +# saved caches +saved_caches_directory: tmp/var/lib/cassandra/saved_caches + +# Size to allow commitlog to grow to before creating a new segment +commitlog_rotation_threshold_in_mb: 128 + +# commitlog_sync may be either "periodic" or "batch." +# When in batch mode, Cassandra won't ack writes until the commit log +# has been fsynced to disk. It will wait up to +# CommitLogSyncBatchWindowInMS milliseconds for other writes, before +# performing the sync. +commitlog_sync: periodic + +# the other option is "timed," where writes may be acked immediately +# and the CommitLog is simply synced every commitlog_sync_period_in_ms +# milliseconds. +commitlog_sync_period_in_ms: 10000 + +# Addresses of hosts that are deemed contact points. +# Cassandra nodes use this list of hosts to find each other and learn +# the topology of the ring. You must change this if you are running +# multiple nodes! +seed_provider: + - class_name: org.apache.cassandra.locator.SimpleSeedProvider + parameters: + - seeds: "127.0.0.1" +# Access mode. mmapped i/o is substantially faster, but only practical on +# a 64bit machine (which notably does not include EC2 "small" instances) +# or relatively small datasets. "auto", the safe choice, will enable +# mmapping on a 64bit JVM. Other values are "mmap", "mmap_index_only" +# (which may allow you to get part of the benefits of mmap on a 32bit +# machine by mmapping only index files) and "standard". +# (The buffer size settings that follow only apply to standard, +# non-mmapped i/o.) +disk_access_mode: auto + +# Unlike most systems, in Cassandra writes are faster than reads, so +# you can afford more of those in parallel. A good rule of thumb is 2 +# concurrent reads per processor core. Increase ConcurrentWrites to +# the number of clients writing at once if you enable CommitLogSync + +# CommitLogSyncDelay. --> +concurrent_reads: 2 +concurrent_writes: 4 + +# This sets the amount of memtable flush writer threads. These will +# be blocked by disk io, and each one will hold a memtable in memory +# while blocked. If you have a large heap and many data directories, +# you can increase this value for better flush performance. +# By default this will be set to the amount of data directories defined. +#memtable_flush_writers: 1 + +# Buffer size to use when performing contiguous column slices. +# Increase this to the size of the column slices you typically perform +sliced_buffer_size_in_kb: 64 + +# TCP port, for commands and data +storage_port: 7000 + +# Address to bind to and tell other Cassandra nodes to connect to. You +# _must_ change this if you want multiple nodes to be able to +# communicate! +# +# Leaving it blank leaves it up to InetAddress.getLocalHost(). This +# will always do the Right Thing *if* the node is properly configured +# (hostname, name resolution, etc), and the Right Thing is to use the +# address associated with the hostname (it might not be). +# +# Setting this to 0.0.0.0 is always wrong. +listen_address: 127.0.0.1 + +# The address to bind the Thrift RPC service to -- clients connect +# here. Unlike ListenAddress above, you *can* specify 0.0.0.0 here if +# you want Thrift to listen on all interfaces. +# +# Leaving this blank has the same effect it does for ListenAddress, +# (i.e. it will be based on the configured hostname of the node). +rpc_address: localhost +# port for Thrift to listen for clients on +rpc_port: 2204 + +# enable or disable keepalive on rpc connections +rpc_keepalive: true + +# uncomment to set socket buffer sizes on rpc connections +# rpc_send_buff_size_in_bytes: +# rpc_recv_buff_size_in_bytes: + +# Frame size for thrift (maximum field length). +# 0 disables TFramedTransport in favor of TSocket. This option +# is deprecated; we strongly recommend using Framed mode. +thrift_framed_transport_size_in_mb: 15 + +# The max length of a thrift message, including all fields and +# internal thrift overhead. +thrift_max_message_length_in_mb: 16 + +# Whether or not to take a snapshot before each compaction. Be +# careful using this option, since Cassandra won't clean up the +# snapshots for you. Mostly useful if you're paranoid when there +# is a data format change. +snapshot_before_compaction: false + +# change this to increase the compaction thread's priority. In java, 1 is the +# lowest priority and that is our default. +# compaction_thread_priority: 1 + +# The threshold size in megabytes the binary memtable must grow to, +# before it's submitted for flushing to disk. +binary_memtable_throughput_in_mb: 256 + +# Add column indexes to a row after its contents reach this size. +# Increase if your column values are large, or if you have a very large +# number of columns. The competing causes are, Cassandra has to +# deserialize this much of the row to read a single column, so you want +# it to be small - at least if you do many partial-row reads - but all +# the index data is read for each access, so you don't want to generate +# that wastefully either. +column_index_size_in_kb: 64 + +# Size limit for rows being compacted in memory. Larger rows will spill +# over to disk and use a slower two-pass compaction process. A message +# will be logged specifying the row key. +in_memory_compaction_limit_in_mb: 16 + +# Time to wait for a reply from other nodes before failing the command +rpc_timeout_in_ms: 10000 + +# phi value that must be reached for a host to be marked down. +# most users should never need to adjust this. +# phi_convict_threshold: 8 + +# endpoint_snitch -- Set this to a class that implements +# IEndpointSnitch, which will let Cassandra know enough +# about your network topology to route requests efficiently. +# Out of the box, Cassandra provides +# - org.apache.cassandra.locator.SimpleSnitch: +# Treats Strategy order as proximity. This improves cache locality +# when disabling read repair, which can further improve throughput. +# - org.apache.cassandra.locator.RackInferringSnitch: +# Proximity is determined by rack and data center, which are +# assumed to correspond to the 3rd and 2nd octet of each node's +# IP address, respectively +# org.apache.cassandra.locator.PropertyFileSnitch: +# - Proximity is determined by rack and data center, which are +# explicitly configured in cassandra-rack.properties. +endpoint_snitch: org.apache.cassandra.locator.SimpleSnitch + +# dynamic_snitch -- This boolean controls whether the above snitch is +# wrapped with a dynamic snitch, which will monitor read latencies +# and avoid reading from hosts that have slowed (due to compaction, +# for instance) +dynamic_snitch: true +# controls how often to perform the more expensive part of host score +# calculation +dynamic_snitch_update_interval_in_ms: 100 +# controls how often to reset all host scores, allowing a bad host to +# possibly recover +dynamic_snitch_reset_interval_in_ms: 600000 +# if set greater than zero and read_repair_chance is < 1.0, this will allow +# 'pinning' of replicas to hosts in order to increase cache capacity. +# The badness threshold will control how much worse the pinned host has to be +# before the dynamic snitch will prefer other replicas over it. This is +# expressed as a double which represents a percentage. +dynamic_snitch_badness_threshold: 0.0 + +# request_scheduler -- Set this to a class that implements +# RequestScheduler, which will schedule incoming client requests +# according to the specific policy. This is useful for multi-tenancy +# with a single Cassandra cluster. +# NOTE: This is specifically for requests from the client and does +# not affect inter node communication. +# org.apache.cassandra.scheduler.NoScheduler - No scheduling takes place +# org.apache.cassandra.scheduler.RoundRobinScheduler - Round robin of +# client requests to a node with a separate queue for each +# request_scheduler_id. The scheduler is further customized by +# request_scheduler_options as described below. +request_scheduler: org.apache.cassandra.scheduler.NoScheduler + +encryption_options: + internode_encryption: none + keystore: conf/.keystore + keystore_password: cassandra + truststore: conf/.truststore + truststore_password: cassandra +# Scheduler Options vary based on the type of scheduler +# NoScheduler - Has no options +# RoundRobin +# - throttle_limit -- The throttle_limit is the number of in-flight +# requests per client. Requests beyond +# that limit are queued up until +# running requests can complete. +# The value of 80 here is twice the number of +# concurrent_reads + concurrent_writes. +# - default_weight -- default_weight is optional and allows for +# overriding the default which is 1. +# - weights -- Weights are optional and will default to 1 or the +# overridden default_weight. The weight translates into how +# many requests are handled during each turn of the +# RoundRobin, based on the scheduler id. +# +# request_scheduler_options: +# throttle_limit: 80 +# default_weight: 5 +# weights: +# Keyspace1: 1 +# Keyspace2: 5 + +# request_scheduler_id -- An identifer based on which to perform +# the request scheduling. Currently the only valid option is keyspace. +# request_scheduler_id: keyspace + +# The Index Interval determines how large the sampling of row keys +# is for a given SSTable. The larger the sampling, the more effective +# the index is at the cost of space. +index_interval: 128 + +# A ColumnFamily is the Cassandra concept closest to a relational table. +# +# Keyspaces are separate groups of ColumnFamilies. Except in very +# unusual circumstances you will have one Keyspace per application. +# +# Keyspace required parameters: +# - name: name of the keyspace; "system" and "definitions" are +# reserved for Cassandra Internals. +# - replica_placement_strategy: the class that determines how replicas +# are distributed among nodes. Contains both the class as well as +# configuration information. Must extend AbstractReplicationStrategy. +# Out of the box, Cassandra provides +# * org.apache.cassandra.locator.SimpleStrategy +# * org.apache.cassandra.locator.NetworkTopologyStrategy +# * org.apache.cassandra.locator.OldNetworkTopologyStrategy +# +# SimpleStrategy merely places the first +# replica at the node whose token is closest to the key (as determined +# by the Partitioner), and additional replicas on subsequent nodes +# along the ring in increasing Token order. +# +# With NetworkTopologyStrategy, +# for each datacenter, you can specify how many replicas you want +# on a per-keyspace basis. Replicas are placed on different racks +# within each DC, if possible. This strategy also requires rack aware +# snitch, such as RackInferringSnitch or PropertyFileSnitch. +# An example: +# - name: Keyspace1 +# replica_placement_strategy: org.apache.cassandra.locator.NetworkTopologyStrategy +# strategy_options: +# DC1 : 3 +# DC2 : 2 +# DC3 : 1 +# +# OldNetworkToplogyStrategy [formerly RackAwareStrategy] +# places one replica in each of two datacenters, and the third on a +# different rack in in the first. Additional datacenters are not +# guaranteed to get a replica. Additional replicas after three are placed +# in ring order after the third without regard to rack or datacenter. +# +# - replication_factor: Number of replicas of each row +# - column_families: column families associated with this keyspace +# +# ColumnFamily required parameters: +# - name: name of the ColumnFamily. Must not contain the character "-". +# - compare_with: tells Cassandra how to sort the columns for slicing +# operations. The default is BytesType, which is a straightforward +# lexical comparison of the bytes in each column. Other options are +# AsciiType, UTF8Type, LexicalUUIDType, TimeUUIDType, LongType, +# and IntegerType (a generic variable-length integer type). +# You can also specify the fully-qualified class name to a class of +# your choice extending org.apache.cassandra.db.marshal.AbstractType. +# +# ColumnFamily optional parameters: +# - keys_cached: specifies the number of keys per sstable whose +# locations we keep in memory in "mostly LRU" order. (JUST the key +# locations, NOT any column values.) Specify a fraction (value less +# than 1) or an absolute number of keys to cache. Defaults to 200000 +# keys. +# - rows_cached: specifies the number of rows whose entire contents we +# cache in memory. Do not use this on ColumnFamilies with large rows, +# or ColumnFamilies with high write:read ratios. Specify a fraction +# (value less than 1) or an absolute number of rows to cache. +# Defaults to 0. (i.e. row caching is off by default) +# - comment: used to attach additional human-readable information about +# the column family to its definition. +# - read_repair_chance: specifies the probability with which read +# repairs should be invoked on non-quorum reads. must be between 0 +# and 1. defaults to 1.0 (always read repair). +# - gc_grace_seconds: specifies the time to wait before garbage +# collecting tombstones (deletion markers). defaults to 864000 (10 +# days). See http://wiki.apache.org/cassandra/DistributedDeletes +# - default_validation_class: specifies a validator class to use for +# validating all the column values in the CF. +# - min_compaction_threshold: the minimum number of SSTables needed +# to start a minor compaction. increasing this will cause minor +# compactions to start less frequently and be more intensive. setting +# this to 0 disables minor compactions. defaults to 4. +# - max_compaction_threshold: the maximum number of SSTables allowed +# before a minor compaction is forced. decreasing this will cause +# minor compactions to start more frequently and be less intensive. +# setting this to 0 disables minor compactions. defaults to 32. +# - row_cache_save_period_in_seconds: number of seconds between saving +# row caches. The row caches can be saved periodically and if one +# exists on startup it will be loaded. +# - key_cache_save_period_in_seconds: number of seconds between saving +# key caches. The key caches can be saved periodically and if one +# exists on startup it will be loaded. +# - memtable_flush_after_mins: The maximum time to leave a dirty table +# unflushed. This should be large enough that it won't cause a flush +# storm of all memtables during periods of inactivity. +# - memtable_throughput_in_mb: The maximum size of the memtable before +# it is flushed. If undefined, 1/8 * heapsize will be used. +# - memtable_operations_in_millions: Number of operations in millions +# before the memtable is flushed. If undefined, throughput / 64 * 0.3 +# will be used. +# +# NOTE: this keyspace definition is for demonstration purposes only. +# Cassandra will not load these definitions during startup. See +# http://wiki.apache.org/cassandra/FAQ#no_keyspaces for an explanation. + From e129f63601d364421ea5cec8333464443153daf4 Mon Sep 17 00:00:00 2001 From: Felipe Sere Date: Mon, 29 Aug 2011 09:36:25 +0200 Subject: [PATCH 7/8] Move EmbeddedServerConfigurator to the hector/test submodule --- .../prettyprint/hector}/testutils/EmbeddedServerConfigurator.java | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename {core/src/main/java/me/prettyprint/cassandra => test/src/main/java/me/prettyprint/hector}/testutils/EmbeddedServerConfigurator.java (100%) diff --git a/core/src/main/java/me/prettyprint/cassandra/testutils/EmbeddedServerConfigurator.java b/test/src/main/java/me/prettyprint/hector/testutils/EmbeddedServerConfigurator.java similarity index 100% rename from core/src/main/java/me/prettyprint/cassandra/testutils/EmbeddedServerConfigurator.java rename to test/src/main/java/me/prettyprint/hector/testutils/EmbeddedServerConfigurator.java From 53c0fa1261ef3d6b6878ed011fc477f1a1102e20 Mon Sep 17 00:00:00 2001 From: Felipe Sere Date: Mon, 29 Aug 2011 10:46:01 +0200 Subject: [PATCH 8/8] Updated constructor tests to match new structure --- .../AdvancedConfiguratorConstructorTest.java | 4 ++-- .../utils/EmbeddedServerTests/DefaultConstructorTest.java | 6 +++--- .../ExternalYamlConfiguratorConstructorTest.java | 4 ++-- .../SimpleConfiguratorConstructorTest.java | 4 ++-- .../hector/testutils/EmbeddedServerConfigurator.java | 3 +-- .../prettyprint/hector}/testutils/cassandra.yaml.template | 0 6 files changed, 10 insertions(+), 11 deletions(-) rename {core/src/main/resources/me/prettyprint/cassandra => test/src/main/resources/me/prettyprint/hector}/testutils/cassandra.yaml.template (100%) diff --git a/core/src/test/java/me/prettyprint/cassandra/utils/EmbeddedServerTests/AdvancedConfiguratorConstructorTest.java b/core/src/test/java/me/prettyprint/cassandra/utils/EmbeddedServerTests/AdvancedConfiguratorConstructorTest.java index bc0067568..a4d8d1482 100644 --- a/core/src/test/java/me/prettyprint/cassandra/utils/EmbeddedServerTests/AdvancedConfiguratorConstructorTest.java +++ b/core/src/test/java/me/prettyprint/cassandra/utils/EmbeddedServerTests/AdvancedConfiguratorConstructorTest.java @@ -2,8 +2,8 @@ import me.prettyprint.cassandra.service.CassandraHostConfigurator; import me.prettyprint.cassandra.service.ThriftCluster; -import me.prettyprint.cassandra.testutils.EmbeddedServerConfigurator; -import me.prettyprint.cassandra.testutils.EmbeddedServerHelper; +import me.prettyprint.hector.testutils.EmbeddedServerConfigurator; +import me.prettyprint.hector.testutils.EmbeddedServerHelper; import org.apache.cassandra.thrift.Cassandra; import org.apache.cassandra.utils.EstimatedHistogram; import org.junit.AfterClass; diff --git a/core/src/test/java/me/prettyprint/cassandra/utils/EmbeddedServerTests/DefaultConstructorTest.java b/core/src/test/java/me/prettyprint/cassandra/utils/EmbeddedServerTests/DefaultConstructorTest.java index c041ea9c5..2abaaed2d 100644 --- a/core/src/test/java/me/prettyprint/cassandra/utils/EmbeddedServerTests/DefaultConstructorTest.java +++ b/core/src/test/java/me/prettyprint/cassandra/utils/EmbeddedServerTests/DefaultConstructorTest.java @@ -1,8 +1,8 @@ package me.prettyprint.cassandra.utils.EmbeddedServerTests; -import me.prettyprint.cassandra.testutils.EmbeddedSchemaLoader; -import me.prettyprint.cassandra.testutils.EmbeddedServerConfigurator; -import me.prettyprint.cassandra.testutils.EmbeddedServerHelper; +import me.prettyprint.hector.testutils.EmbeddedSchemaLoader; +import me.prettyprint.hector.testutils.EmbeddedServerConfigurator; +import me.prettyprint.hector.testutils.EmbeddedServerHelper; import org.apache.cassandra.service.EmbeddedCassandraService; import org.slf4j.LoggerFactory; diff --git a/core/src/test/java/me/prettyprint/cassandra/utils/EmbeddedServerTests/ExternalYamlConfiguratorConstructorTest.java b/core/src/test/java/me/prettyprint/cassandra/utils/EmbeddedServerTests/ExternalYamlConfiguratorConstructorTest.java index 748d6c800..319277bec 100644 --- a/core/src/test/java/me/prettyprint/cassandra/utils/EmbeddedServerTests/ExternalYamlConfiguratorConstructorTest.java +++ b/core/src/test/java/me/prettyprint/cassandra/utils/EmbeddedServerTests/ExternalYamlConfiguratorConstructorTest.java @@ -2,8 +2,8 @@ import me.prettyprint.cassandra.service.CassandraHostConfigurator; import me.prettyprint.cassandra.service.ThriftCluster; -import me.prettyprint.cassandra.testutils.EmbeddedServerConfigurator; -import me.prettyprint.cassandra.testutils.EmbeddedServerHelper; +import me.prettyprint.hector.testutils.EmbeddedServerConfigurator; +import me.prettyprint.hector.testutils.EmbeddedServerHelper; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Test; diff --git a/core/src/test/java/me/prettyprint/cassandra/utils/EmbeddedServerTests/SimpleConfiguratorConstructorTest.java b/core/src/test/java/me/prettyprint/cassandra/utils/EmbeddedServerTests/SimpleConfiguratorConstructorTest.java index 900cc31f9..ae94159d1 100644 --- a/core/src/test/java/me/prettyprint/cassandra/utils/EmbeddedServerTests/SimpleConfiguratorConstructorTest.java +++ b/core/src/test/java/me/prettyprint/cassandra/utils/EmbeddedServerTests/SimpleConfiguratorConstructorTest.java @@ -1,7 +1,7 @@ package me.prettyprint.cassandra.utils.EmbeddedServerTests; -import me.prettyprint.cassandra.testutils.EmbeddedServerConfigurator; -import me.prettyprint.cassandra.testutils.EmbeddedServerHelper; +import me.prettyprint.hector.testutils.EmbeddedServerConfigurator; +import me.prettyprint.hector.testutils.EmbeddedServerHelper; import org.junit.Test; import java.util.concurrent.TimeUnit; diff --git a/test/src/main/java/me/prettyprint/hector/testutils/EmbeddedServerConfigurator.java b/test/src/main/java/me/prettyprint/hector/testutils/EmbeddedServerConfigurator.java index 2ed874a44..9b408fb28 100644 --- a/test/src/main/java/me/prettyprint/hector/testutils/EmbeddedServerConfigurator.java +++ b/test/src/main/java/me/prettyprint/hector/testutils/EmbeddedServerConfigurator.java @@ -1,6 +1,5 @@ -package me.prettyprint.cassandra.testutils; +package me.prettyprint.hector.testutils; -import me.prettyprint.cassandra.service.template.MappedColumnFamilyResult; import org.yaml.snakeyaml.Yaml; import java.io.File; diff --git a/core/src/main/resources/me/prettyprint/cassandra/testutils/cassandra.yaml.template b/test/src/main/resources/me/prettyprint/hector/testutils/cassandra.yaml.template similarity index 100% rename from core/src/main/resources/me/prettyprint/cassandra/testutils/cassandra.yaml.template rename to test/src/main/resources/me/prettyprint/hector/testutils/cassandra.yaml.template