Skip to content

Commit

Permalink
implement #630 switch to otj pg embedded (#633)
Browse files Browse the repository at this point in the history 
* #630 switch to otj-pg-embedded: at the unit test level

* #630 launch postgres with the expected conf

* #630 initial fix for start pg instance

* #630 implement stop postgresql
  • Loading branch information
@syjer @cbellone
syjer authored and cbellone committed Apr 28, 2019
1 parent 43a315d commit 0f24e5c
Show file tree
Hide file tree
Showing 2 changed files with 97 additions and 52 deletions.
118 changes: 83 additions & 35 deletions build.gradle
View file
Original file line number Diff line number Diff line change
@@ -1,3 +1,8 @@
import com.opentable.db.postgres.embedded.EmbeddedPostgres
import com.opentable.db.postgres.embedded.PgBinaryResolver
import org.apache.commons.io.FileUtils
import org.apache.commons.io.IOUtils
import org.apache.commons.lang3.SystemUtils
import org.apache.tools.ant.filters.ReplaceTokens
import org.springframework.jdbc.core.JdbcTemplate

Expand All @@ -8,10 +13,12 @@ import java.time.ZoneId
import java.time.ZonedDateTime
import java.time.format.DateTimeFormatter

import static java.lang.String.format

buildscript {

dependencies {
classpath 'ru.yandex.qatools.embed:postgresql-embedded:2.9'
classpath 'com.opentable.components:otj-pg-embedded:0.13.1'
classpath 'org.postgresql:postgresql:42.2.5'
classpath "org.springframework:spring-jdbc:$springVersion"
}
Expand Down Expand Up @@ -141,7 +148,7 @@ dependencies {
compile 'org.imgscalr:imgscalr-lib:4.2'
compile 'org.aspectj:aspectjweaver:1.9.2'

testCompile "ru.yandex.qatools.embed:postgresql-embedded:2.9" //we leave it to the v2.9 for now, since some user had issues on Windows
testCompile 'com.opentable.components:otj-pg-embedded:0.13.1'

compileOnly "javax.servlet:javax.servlet-api:4.0.1"
testCompile "javax.servlet:javax.servlet-api:4.0.1"
Expand Down Expand Up @@ -308,47 +315,88 @@ task clever(type: Copy) {
dependsOn build
}

import ru.yandex.qatools.embed.postgresql.EmbeddedPostgres
import ru.yandex.qatools.embed.postgresql.distribution.Version.Main
import org.postgresql.ds.PGSimpleDataSource

task startEmbeddedPgSQL {
doLast {
final pgsqlPath = Paths.get(".", "alfio-itest")
def pgsqlPath = Paths.get(System.getProperty("user.dir"), "alfio-itest")
Files.createDirectories(pgsqlPath)
final tmpDataDir = Files.createTempDirectory(pgsqlPath, "alfio-data")
final postgres = new EmbeddedPostgres(Main.PRODUCTION, tmpDataDir.normalize().toAbsolutePath().toString())
postgres.start(EmbeddedPostgres.cachedRuntimeConfig(Paths.get(System.getProperty("java.io.tmpdir"), "pgembed")),
"localhost", 5432, "alfio", "admin", "password", Arrays.asList("-E", "SQL_ASCII", "--locale=C", "--lc-collate=C", "--lc-ctype=C"))

postgres.getProcess().ifPresent({
final pid = it.getProcessId()
Files.write(Paths.get(".", "alfio-itest", "pgsql-pid"), Arrays.asList(Long.toString(pid)))
System.out.println("Launched pgsql with pid " + pid)

final dataSource = new PGSimpleDataSource()
dataSource.setUrl("jdbc:postgresql://localhost:5432/postgres");
dataSource.setUser("admin")
dataSource.setPassword("password")
final jdbc = new JdbcTemplate(dataSource);
jdbc.execute("CREATE ROLE postgres LOGIN PASSWORD 'password' NOSUPERUSER INHERIT CREATEDB CREATEROLE REPLICATION")
//todo: check if the following grants are necessary
jdbc.execute("GRANT pg_monitor TO postgres")
jdbc.execute("GRANT pg_read_all_settings TO postgres")
jdbc.execute("GRANT pg_read_all_stats TO postgres")
jdbc.execute("GRANT pg_signal_backend TO postgres")
jdbc.execute("GRANT pg_stat_scan_tables TO postgres")
})
def tmpDataDir = Files.createTempDirectory(pgsqlPath, "alfio-data")

def binDir = "alfio-pg-bin"+(UUID.randomUUID().toString())
def postgresBinariesDir = Files.createDirectories(pgsqlPath.resolve(binDir)).toAbsolutePath().toFile()

def postgres = EmbeddedPostgres.builder()
.setPort(5432)
.setDataDirectory(tmpDataDir)
.setOverrideWorkingDirectory(postgresBinariesDir)
// we are doing this because there is an unfortunate interplay with the persistence of the gradle daemon and
// EmbeddedPostgres.PREPARE_BINARIES (which is a static field)
// this cause to reuse a wrong directory
.setPgBinaryResolver(new PgBinaryResolver() {
@Override
InputStream getPgBinary(String system, String machineHardware) {
return EmbeddedPostgres.class.getResourceAsStream(format("/postgresql-%s-%s.txz", system, machineHardware));
}
})
.start()

// create a new admin, and then rename the user "postgres" as we don't want to have the superuser roles
def jdbc = new JdbcTemplate(postgres.getDatabase("postgres", "postgres"))
jdbc.execute("CREATE ROLE tmpadmin LOGIN PASSWORD 'password' SUPERUSER")

jdbc = new JdbcTemplate(postgres.getDatabase("tmpadmin", "postgres"))
jdbc.execute("ALTER USER postgres RENAME TO admin")
//
//todo: check if the following grants are necessary
jdbc.execute("CREATE ROLE postgres LOGIN PASSWORD 'password' NOSUPERUSER INHERIT CREATEDB CREATEROLE REPLICATION")
jdbc.execute("GRANT pg_monitor TO postgres")
jdbc.execute("GRANT pg_read_all_settings TO postgres")
jdbc.execute("GRANT pg_read_all_stats TO postgres")
jdbc.execute("GRANT pg_signal_backend TO postgres")
jdbc.execute("GRANT pg_stat_scan_tables TO postgres")
jdbc.execute("CREATE DATABASE alfio WITH OWNER postgres")
//
Files.write(Paths.get(".", "alfio-itest", "pgsql-descriptor"), Arrays.asList(tmpDataDir.toAbsolutePath().toFile().toString(), postgresBinariesDir.toString()))
System.out.println("Launched pgsql")
}
}

// code imported from EmbeddedPostgres :D
String pgBin(File pgDirBase, String binaryName) {
final pgDir = pgDirBase.listFiles(new FileFilter() {
@Override
boolean accept(File file) {
return file.getName().startsWith("PG-") && file.isDirectory()
}
})[0]
final String extension = SystemUtils.IS_OS_WINDOWS ? ".exe" : ""
return new File(pgDir, "bin/" + binaryName + extension).getPath()
}

void system(String... command) {
final ProcessBuilder builder = new ProcessBuilder(command)
final Process process = builder.start()
if (0 != process.waitFor()) {
throw new IllegalStateException(String.format("Process %s failed%n%s", Arrays.asList(command), IOUtils.toString(process.getErrorStream())))
}
}

void pgCtl(File binDir, File dataDir, String action) {
system(pgBin(binDir, "pg_ctl"), "-D", dataDir.getPath(), action, "-m", "fast", "-t", "5", "-w")
}
//

task stopEmbeddedPgSQL {
doLast {
final pidFile = Paths.get(".", "alfio-itest", "pgsql-pid");
final pid = Files.readAllLines(pidFile).get(0)
Files.deleteIfExists(pidFile)
Runtime.runtime.exec("kill -9 " + pid)
System.out.println("Killed pgsql with pid " + pid)
def descriptor = Paths.get(".", "alfio-itest", "pgsql-descriptor")
def r = Files.readAllLines(descriptor)
def dataDir = new File(r.get(0))
def binariesDir = new File(r.get(1))

pgCtl(binariesDir, dataDir, "stop")
Files.deleteIfExists(descriptor)
FileUtils.deleteDirectory(dataDir)
FileUtils.deleteDirectory(binariesDir)
System.out.println("Stopped postgresql")
}
}

Expand Down
31 changes: 14 additions & 17 deletions src/test/java/alfio/TestConfiguration.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,14 +20,14 @@
import alfio.manager.FileDownloadManager;
import alfio.test.util.IntegrationTestUtil;
import alfio.util.BaseIntegrationTest;
import com.opentable.db.postgres.embedded.EmbeddedPostgres;
import com.zaxxer.hikari.HikariDataSource;
import org.apache.commons.io.FileUtils;
import org.springframework.beans.factory.config.PropertyPlaceholderConfigurer;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.Profile;
import org.springframework.core.io.ByteArrayResource;
import ru.yandex.qatools.embed.postgresql.EmbeddedPostgres;


import javax.annotation.PreDestroy;
import javax.sql.DataSource;
Expand All @@ -41,17 +41,21 @@
import java.time.ZonedDateTime;
import java.util.Properties;

import static ru.yandex.qatools.embed.postgresql.distribution.Version.Main.PRODUCTION;


@Configuration
public class TestConfiguration {

private EmbeddedPostgres postgres;

private final String POSTGRES_USERNAME = "postgres";
private final String POSTGRES_PASSWORD = "postgres";
private final String POSTGRES_DB = "postgres";

@Bean
@Profile("!travis")
public PlatformProvider getCloudProvider(EmbeddedPostgres postgres) {
IntegrationTestUtil.generateDBConfig(postgres.getConnectionUrl().orElseThrow(IllegalArgumentException::new), EmbeddedPostgres.DEFAULT_USER, EmbeddedPostgres.DEFAULT_PASSWORD)
IntegrationTestUtil.generateDBConfig(postgres.getJdbcUrl(POSTGRES_USERNAME, POSTGRES_DB), POSTGRES_USERNAME, POSTGRES_PASSWORD)
.forEach(System::setProperty);
return PlatformProvider.DEFAULT;
}
Expand All @@ -60,9 +64,9 @@ public PlatformProvider getCloudProvider(EmbeddedPostgres postgres) {
@Profile("!travis")
public DataSource getDataSource(EmbeddedPostgres postgres) {
HikariDataSource dataSource = new HikariDataSource();
dataSource.setJdbcUrl(postgres.getConnectionUrl().orElseThrow(IllegalArgumentException::new));
dataSource.setUsername(EmbeddedPostgres.DEFAULT_USER);
dataSource.setPassword(EmbeddedPostgres.DEFAULT_PASSWORD);
dataSource.setJdbcUrl(postgres.getJdbcUrl(POSTGRES_USERNAME, POSTGRES_DB));
dataSource.setUsername(POSTGRES_USERNAME);
dataSource.setPassword(POSTGRES_PASSWORD);
dataSource.setDriverClassName("org.postgresql.Driver");
dataSource.setMaximumPoolSize(5);
return dataSource;
Expand All @@ -74,21 +78,14 @@ public EmbeddedPostgres postgres() throws IOException {
Path pgsqlPath = Paths.get(".", "alfio-itest");
Files.createDirectories(pgsqlPath);
Path tmpDataDir = Files.createTempDirectory(pgsqlPath, "alfio-data");
postgres = new EmbeddedPostgres(PRODUCTION, tmpDataDir.normalize().toAbsolutePath().toString());
postgres.start(EmbeddedPostgres.cachedRuntimeConfig(Paths.get(System.getProperty("java.io.tmpdir"), "pgembed")));
Runtime.getRuntime().addShutdownHook(new Thread(() -> {
try {
FileUtils.deleteDirectory(tmpDataDir.normalize().toAbsolutePath().toFile());
} catch (IOException e) {
}
}));
postgres = EmbeddedPostgres.builder().setDataDirectory(tmpDataDir).start();
return postgres;
}

@PreDestroy
public void shutdown() {
public void shutdown() throws IOException {
if (postgres != null) {
postgres.stop();
postgres.close();
}
}

Expand Down

0 comments on commit 0f24e5c

Please sign in to comment.