From 0f24e5c53a8c8774a8c14051a6b19fc36e089f18 Mon Sep 17 00:00:00 2001 From: Sylvain Jermini Date: Sun, 28 Apr 2019 15:32:28 +0200 Subject: [PATCH] implement #630 switch to otj pg embedded (#633) * #630 switch to otj-pg-embedded: at the unit test level * #630 launch postgres with the expected conf * #630 initial fix for start pg instance * #630 implement stop postgresql --- build.gradle | 118 +++++++++++++++------ src/test/java/alfio/TestConfiguration.java | 31 +++--- 2 files changed, 97 insertions(+), 52 deletions(-) diff --git a/build.gradle b/build.gradle index 84883bd6b7..1e236f307b 100644 --- a/build.gradle +++ b/build.gradle @@ -1,3 +1,8 @@ +import com.opentable.db.postgres.embedded.EmbeddedPostgres +import com.opentable.db.postgres.embedded.PgBinaryResolver +import org.apache.commons.io.FileUtils +import org.apache.commons.io.IOUtils +import org.apache.commons.lang3.SystemUtils import org.apache.tools.ant.filters.ReplaceTokens import org.springframework.jdbc.core.JdbcTemplate @@ -8,10 +13,12 @@ import java.time.ZoneId import java.time.ZonedDateTime import java.time.format.DateTimeFormatter +import static java.lang.String.format + buildscript { dependencies { - classpath 'ru.yandex.qatools.embed:postgresql-embedded:2.9' + classpath 'com.opentable.components:otj-pg-embedded:0.13.1' classpath 'org.postgresql:postgresql:42.2.5' classpath "org.springframework:spring-jdbc:$springVersion" } @@ -141,7 +148,7 @@ dependencies { compile 'org.imgscalr:imgscalr-lib:4.2' compile 'org.aspectj:aspectjweaver:1.9.2' - testCompile "ru.yandex.qatools.embed:postgresql-embedded:2.9" //we leave it to the v2.9 for now, since some user had issues on Windows + testCompile 'com.opentable.components:otj-pg-embedded:0.13.1' compileOnly "javax.servlet:javax.servlet-api:4.0.1" testCompile "javax.servlet:javax.servlet-api:4.0.1" @@ -308,47 +315,88 @@ task clever(type: Copy) { dependsOn build } -import ru.yandex.qatools.embed.postgresql.EmbeddedPostgres -import ru.yandex.qatools.embed.postgresql.distribution.Version.Main -import org.postgresql.ds.PGSimpleDataSource - task startEmbeddedPgSQL { doLast { - final pgsqlPath = Paths.get(".", "alfio-itest") + def pgsqlPath = Paths.get(System.getProperty("user.dir"), "alfio-itest") Files.createDirectories(pgsqlPath) - final tmpDataDir = Files.createTempDirectory(pgsqlPath, "alfio-data") - final postgres = new EmbeddedPostgres(Main.PRODUCTION, tmpDataDir.normalize().toAbsolutePath().toString()) - postgres.start(EmbeddedPostgres.cachedRuntimeConfig(Paths.get(System.getProperty("java.io.tmpdir"), "pgembed")), - "localhost", 5432, "alfio", "admin", "password", Arrays.asList("-E", "SQL_ASCII", "--locale=C", "--lc-collate=C", "--lc-ctype=C")) - - postgres.getProcess().ifPresent({ - final pid = it.getProcessId() - Files.write(Paths.get(".", "alfio-itest", "pgsql-pid"), Arrays.asList(Long.toString(pid))) - System.out.println("Launched pgsql with pid " + pid) - - final dataSource = new PGSimpleDataSource() - dataSource.setUrl("jdbc:postgresql://localhost:5432/postgres"); - dataSource.setUser("admin") - dataSource.setPassword("password") - final jdbc = new JdbcTemplate(dataSource); - jdbc.execute("CREATE ROLE postgres LOGIN PASSWORD 'password' NOSUPERUSER INHERIT CREATEDB CREATEROLE REPLICATION") - //todo: check if the following grants are necessary - jdbc.execute("GRANT pg_monitor TO postgres") - jdbc.execute("GRANT pg_read_all_settings TO postgres") - jdbc.execute("GRANT pg_read_all_stats TO postgres") - jdbc.execute("GRANT pg_signal_backend TO postgres") - jdbc.execute("GRANT pg_stat_scan_tables TO postgres") - }) + def tmpDataDir = Files.createTempDirectory(pgsqlPath, "alfio-data") + + def binDir = "alfio-pg-bin"+(UUID.randomUUID().toString()) + def postgresBinariesDir = Files.createDirectories(pgsqlPath.resolve(binDir)).toAbsolutePath().toFile() + + def postgres = EmbeddedPostgres.builder() + .setPort(5432) + .setDataDirectory(tmpDataDir) + .setOverrideWorkingDirectory(postgresBinariesDir) + // we are doing this because there is an unfortunate interplay with the persistence of the gradle daemon and + // EmbeddedPostgres.PREPARE_BINARIES (which is a static field) + // this cause to reuse a wrong directory + .setPgBinaryResolver(new PgBinaryResolver() { + @Override + InputStream getPgBinary(String system, String machineHardware) { + return EmbeddedPostgres.class.getResourceAsStream(format("/postgresql-%s-%s.txz", system, machineHardware)); + } + }) + .start() + + // create a new admin, and then rename the user "postgres" as we don't want to have the superuser roles + def jdbc = new JdbcTemplate(postgres.getDatabase("postgres", "postgres")) + jdbc.execute("CREATE ROLE tmpadmin LOGIN PASSWORD 'password' SUPERUSER") + + jdbc = new JdbcTemplate(postgres.getDatabase("tmpadmin", "postgres")) + jdbc.execute("ALTER USER postgres RENAME TO admin") + // + //todo: check if the following grants are necessary + jdbc.execute("CREATE ROLE postgres LOGIN PASSWORD 'password' NOSUPERUSER INHERIT CREATEDB CREATEROLE REPLICATION") + jdbc.execute("GRANT pg_monitor TO postgres") + jdbc.execute("GRANT pg_read_all_settings TO postgres") + jdbc.execute("GRANT pg_read_all_stats TO postgres") + jdbc.execute("GRANT pg_signal_backend TO postgres") + jdbc.execute("GRANT pg_stat_scan_tables TO postgres") + jdbc.execute("CREATE DATABASE alfio WITH OWNER postgres") + // + Files.write(Paths.get(".", "alfio-itest", "pgsql-descriptor"), Arrays.asList(tmpDataDir.toAbsolutePath().toFile().toString(), postgresBinariesDir.toString())) + System.out.println("Launched pgsql") } } +// code imported from EmbeddedPostgres :D +String pgBin(File pgDirBase, String binaryName) { + final pgDir = pgDirBase.listFiles(new FileFilter() { + @Override + boolean accept(File file) { + return file.getName().startsWith("PG-") && file.isDirectory() + } + })[0] + final String extension = SystemUtils.IS_OS_WINDOWS ? ".exe" : "" + return new File(pgDir, "bin/" + binaryName + extension).getPath() +} + +void system(String... command) { + final ProcessBuilder builder = new ProcessBuilder(command) + final Process process = builder.start() + if (0 != process.waitFor()) { + throw new IllegalStateException(String.format("Process %s failed%n%s", Arrays.asList(command), IOUtils.toString(process.getErrorStream()))) + } +} + +void pgCtl(File binDir, File dataDir, String action) { + system(pgBin(binDir, "pg_ctl"), "-D", dataDir.getPath(), action, "-m", "fast", "-t", "5", "-w") +} +// + task stopEmbeddedPgSQL { doLast { - final pidFile = Paths.get(".", "alfio-itest", "pgsql-pid"); - final pid = Files.readAllLines(pidFile).get(0) - Files.deleteIfExists(pidFile) - Runtime.runtime.exec("kill -9 " + pid) - System.out.println("Killed pgsql with pid " + pid) + def descriptor = Paths.get(".", "alfio-itest", "pgsql-descriptor") + def r = Files.readAllLines(descriptor) + def dataDir = new File(r.get(0)) + def binariesDir = new File(r.get(1)) + + pgCtl(binariesDir, dataDir, "stop") + Files.deleteIfExists(descriptor) + FileUtils.deleteDirectory(dataDir) + FileUtils.deleteDirectory(binariesDir) + System.out.println("Stopped postgresql") } } diff --git a/src/test/java/alfio/TestConfiguration.java b/src/test/java/alfio/TestConfiguration.java index fff1ce01a0..cf59171729 100644 --- a/src/test/java/alfio/TestConfiguration.java +++ b/src/test/java/alfio/TestConfiguration.java @@ -20,14 +20,14 @@ import alfio.manager.FileDownloadManager; import alfio.test.util.IntegrationTestUtil; import alfio.util.BaseIntegrationTest; +import com.opentable.db.postgres.embedded.EmbeddedPostgres; import com.zaxxer.hikari.HikariDataSource; -import org.apache.commons.io.FileUtils; import org.springframework.beans.factory.config.PropertyPlaceholderConfigurer; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Profile; import org.springframework.core.io.ByteArrayResource; -import ru.yandex.qatools.embed.postgresql.EmbeddedPostgres; + import javax.annotation.PreDestroy; import javax.sql.DataSource; @@ -41,17 +41,21 @@ import java.time.ZonedDateTime; import java.util.Properties; -import static ru.yandex.qatools.embed.postgresql.distribution.Version.Main.PRODUCTION; + @Configuration public class TestConfiguration { private EmbeddedPostgres postgres; + private final String POSTGRES_USERNAME = "postgres"; + private final String POSTGRES_PASSWORD = "postgres"; + private final String POSTGRES_DB = "postgres"; + @Bean @Profile("!travis") public PlatformProvider getCloudProvider(EmbeddedPostgres postgres) { - IntegrationTestUtil.generateDBConfig(postgres.getConnectionUrl().orElseThrow(IllegalArgumentException::new), EmbeddedPostgres.DEFAULT_USER, EmbeddedPostgres.DEFAULT_PASSWORD) + IntegrationTestUtil.generateDBConfig(postgres.getJdbcUrl(POSTGRES_USERNAME, POSTGRES_DB), POSTGRES_USERNAME, POSTGRES_PASSWORD) .forEach(System::setProperty); return PlatformProvider.DEFAULT; } @@ -60,9 +64,9 @@ public PlatformProvider getCloudProvider(EmbeddedPostgres postgres) { @Profile("!travis") public DataSource getDataSource(EmbeddedPostgres postgres) { HikariDataSource dataSource = new HikariDataSource(); - dataSource.setJdbcUrl(postgres.getConnectionUrl().orElseThrow(IllegalArgumentException::new)); - dataSource.setUsername(EmbeddedPostgres.DEFAULT_USER); - dataSource.setPassword(EmbeddedPostgres.DEFAULT_PASSWORD); + dataSource.setJdbcUrl(postgres.getJdbcUrl(POSTGRES_USERNAME, POSTGRES_DB)); + dataSource.setUsername(POSTGRES_USERNAME); + dataSource.setPassword(POSTGRES_PASSWORD); dataSource.setDriverClassName("org.postgresql.Driver"); dataSource.setMaximumPoolSize(5); return dataSource; @@ -74,21 +78,14 @@ public EmbeddedPostgres postgres() throws IOException { Path pgsqlPath = Paths.get(".", "alfio-itest"); Files.createDirectories(pgsqlPath); Path tmpDataDir = Files.createTempDirectory(pgsqlPath, "alfio-data"); - postgres = new EmbeddedPostgres(PRODUCTION, tmpDataDir.normalize().toAbsolutePath().toString()); - postgres.start(EmbeddedPostgres.cachedRuntimeConfig(Paths.get(System.getProperty("java.io.tmpdir"), "pgembed"))); - Runtime.getRuntime().addShutdownHook(new Thread(() -> { - try { - FileUtils.deleteDirectory(tmpDataDir.normalize().toAbsolutePath().toFile()); - } catch (IOException e) { - } - })); + postgres = EmbeddedPostgres.builder().setDataDirectory(tmpDataDir).start(); return postgres; } @PreDestroy - public void shutdown() { + public void shutdown() throws IOException { if (postgres != null) { - postgres.stop(); + postgres.close(); } }