john/dbt
1
Fork 0
Code Activity

Replace in-memory ledger with MongoDB

Browse Source
This commit is contained in:
John Ahlroos
2018-06-09 20:45:18 +03:00
parent 0e21b7eee2
commit 0f24c37ac5
17 changed files with 209 additions and 174 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
dbt-core/build.gradle
View File

@@ -1,4 +1,6 @@
dependencies {
compile ratpack.dependency('jdbc-tx')
//compile 'org.mongodb:mongo-java-driver:3.7.1'
compile 'com.gmongo:gmongo:1.2'
compile 'com.fasterxml.jackson.module:jackson-module-jsonSchema:2.9.0'
}

80
dbt-core/src/main/groovy/com/devsoap/dbt/DBTModule.groovy
View File

@@ -1,80 +0,0 @@
/*
* Copyright 2018 Devsoap Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.devsoap.dbt
import com.devsoap.dbt.actions.ExecutorChainAction
import com.devsoap.dbt.actions.LedgerChainAction
import com.devsoap.dbt.config.DBTConfig
import com.devsoap.dbt.data.LedgerData
import com.devsoap.dbt.handlers.ExecutorHandler
import com.devsoap.dbt.handlers.ConfigInfoHandler
import com.devsoap.dbt.handlers.JsonSchemaHandler
import com.devsoap.dbt.handlers.LedgerGetTransactionHandler
import com.devsoap.dbt.handlers.LedgerListTransactionsHandler
import com.devsoap.dbt.handlers.LedgerUpdateTransactionHandler
import com.devsoap.dbt.services.LedgerService
import com.devsoap.dbt.services.TransactionManagerService
import com.fasterxml.jackson.databind.ObjectMapper
import com.google.inject.multibindings.Multibinder
import groovy.util.logging.Slf4j
import ratpack.guice.ConfigurableModule
import ratpack.handling.HandlerDecorator
import ratpack.server.ServerConfig
@Slf4j
class DBTModule extends ConfigurableModule<DBTConfig> {
@Override
protected void configure() {
bind(ObjectMapper)
bind(LedgerChainAction)
bind(LedgerGetTransactionHandler)
bind(LedgerListTransactionsHandler)
bind(LedgerUpdateTransactionHandler)
bind(ExecutorChainAction)
bind(ExecutorHandler)
bind(LedgerData)
bind(LedgerService)
bind(TransactionManagerService)
bind(ConfigInfoHandler)
bind(JsonSchemaHandler)
Multibinder.newSetBinder(binder(), HandlerDecorator).addBinding()
.toInstance(HandlerDecorator.prependHandlers(LedgerChainAction))
Multibinder.newSetBinder(binder(), HandlerDecorator).addBinding()
.toInstance(HandlerDecorator.prependHandlers(ExecutorChainAction))
}
@Override
protected DBTConfig createConfig(ServerConfig serverConfig) {
(DBTConfig) serverConfig.getAsConfigObject('/dbt', DBTConfig)?.getObject() ?: super.createConfig(serverConfig)
}
@Override
protected void defaultConfig(ServerConfig serverConfig, DBTConfig config) {
if(!config.executor.remoteUrl) {
config.executor.remoteUrl = "http://localhost:${serverConfig.port}/${config.executor.path}"
}
if(!config.ledger.remoteUrl) {
config.ledger.remoteUrl = "http://localhost:${serverConfig.port}/${config.ledger.path}"
}
}
}

5
dbt-core/src/main/groovy/com/devsoap/dbt/config/LedgerConfig.groovy
View File

@@ -31,4 +31,9 @@ class LedgerConfig {
* If the ledger is disabled, then this specifies the remote url of the ledger the executors use
*/
String remoteUrl
/**
* Database JDBC Url
*/
String databaseUrl
}

38
dbt-core/src/main/groovy/com/devsoap/dbt/data/BlockTransaction.groovy
View File

@@ -59,42 +59,4 @@ class BlockTransaction implements Serializable {
void commit() {
completed = true
}
// A block in the chain
@ToString
static final class Query implements Serializable {
@JsonProperty(required = true)
String query
String id
String parent
long timeStamp
Map<String, List> result
String resultError
Query() {
// For serialization
}
Query(Query previous, String query) {
this.query = query
timeStamp = new Date().getTime()
parent = previous.id
id = generateHash()
}
Query(BlockTransaction transaction, String query) {
this.query = query
timeStamp = new Date().getTime()
parent = transaction.id
id = generateHash()
}
final String generateHash() {
def digest = MessageDigest.getInstance('SHA-256')
def hash = digest.digest("${parent?:''}$timeStamp$query".getBytes(StandardCharsets.UTF_8))
hash.encodeHex().toString()
}
}
}

21
dbt-core/src/main/groovy/com/devsoap/dbt/data/LedgerData.groovy
View File

@@ -1,21 +0,0 @@
/*
* Copyright 2018 Devsoap Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.devsoap.dbt.data
class LedgerData implements Serializable {
List<BlockTransaction> transactions = new ArrayList<>()
}

46
dbt-core/src/main/groovy/com/devsoap/dbt/data/Query.groovy Normal file
View File

@@ -0,0 +1,46 @@
package com.devsoap.dbt.data
import com.fasterxml.jackson.annotation.JsonProperty
import groovy.transform.ToString
import java.nio.charset.StandardCharsets
import java.security.MessageDigest
@ToString
class Query implements Serializable {
@JsonProperty(required = true)
String query
String id
String parent
long timeStamp
// Column -> Values
Map<String, List<String>> result
String resultError
Query() {
// For serialization
}
Query(Query previous, String query) {
this.query = query
timeStamp = new Date().getTime()
parent = previous.id
id = generateHash()
}
Query(BlockTransaction transaction, String query) {
this.query = query
timeStamp = new Date().getTime()
parent = transaction.id
id = generateHash()
}
final String generateHash() {
def digest = MessageDigest.getInstance('SHA-256')
def hash = digest.digest("${parent?:''}$timeStamp$query".getBytes(StandardCharsets.UTF_8))
hash.encodeHex().toString()
}
}

4
dbt-core/src/main/groovy/com/devsoap/dbt/handlers/ExecutorHandler.groovy
View File

@@ -140,7 +140,7 @@ class ExecutorHandler implements Handler {
}
}
private static Map toMap(ResultSet resultSet) {
private static Map<String,List<String>> toMap(ResultSet resultSet) {
def map = [:]
if(resultSet.last()) {
@@ -156,7 +156,7 @@ class ExecutorHandler implements Handler {
resultSet.beforeFirst()
while(resultSet.next()) {
columnValues << resultSet.getObject(columnIndex)
columnValues << resultSet.getObject(columnIndex).toString()
}
}
}

5
dbt-core/src/main/groovy/com/devsoap/dbt/handlers/LedgerUpdateTransactionHandler.groovy
View File

@@ -17,6 +17,7 @@ package com.devsoap.dbt.handlers
import com.devsoap.dbt.config.DBTConfig
import com.devsoap.dbt.data.BlockTransaction
import com.devsoap.dbt.data.Query
import com.devsoap.dbt.services.LedgerService
import com.fasterxml.jackson.databind.ObjectMapper
import groovy.util.logging.Slf4j
@@ -128,8 +129,8 @@ class LedgerUpdateTransactionHandler implements Handler {
newTransaction.queries.each { q ->
def query = transaction.queries.isEmpty() ?
new BlockTransaction.Query(transaction, q.query) :
new BlockTransaction.Query(transaction.queries.last(), q.query)
new Query(transaction, q.query) :
new Query(transaction.queries.last(), q.query)
query.resultError = q.resultError
query.result = q.result
transaction.queries << query

30
dbt-core/src/main/groovy/com/devsoap/dbt/modules/DBTExecutorModule.groovy Normal file
View File

@@ -0,0 +1,30 @@
package com.devsoap.dbt.modules
import com.devsoap.dbt.actions.ExecutorChainAction
import com.devsoap.dbt.config.DBTConfig
import com.devsoap.dbt.handlers.ExecutorHandler
import com.google.inject.multibindings.Multibinder
import ratpack.handling.HandlerDecorator
import ratpack.server.ServerConfig
class DBTExecutorModule extends DBTModule {
@Override
protected void configure() {
super.configure()
bind(ExecutorChainAction)
bind(ExecutorHandler)
Multibinder.newSetBinder(binder(), HandlerDecorator).addBinding()
.toInstance(HandlerDecorator.prependHandlers(ExecutorChainAction))
}
@Override
protected void defaultConfig(ServerConfig serverConfig, DBTConfig config) {
config.executor.enabled = !config.executor.remoteUrl
if(!config.executor.remoteUrl) {
config.executor.remoteUrl = "http://localhost:${serverConfig.port}/${config.executor.path}"
}
}
}

36
dbt-core/src/main/groovy/com/devsoap/dbt/modules/DBTLedgerModule.groovy Normal file
View File

@@ -0,0 +1,36 @@
package com.devsoap.dbt.modules
import com.devsoap.dbt.actions.LedgerChainAction
import com.devsoap.dbt.config.DBTConfig
import com.devsoap.dbt.handlers.LedgerGetTransactionHandler
import com.devsoap.dbt.handlers.LedgerListTransactionsHandler
import com.devsoap.dbt.handlers.LedgerUpdateTransactionHandler
import com.devsoap.dbt.services.LedgerService
import com.google.inject.multibindings.Multibinder
import ratpack.handling.HandlerDecorator
import ratpack.server.ServerConfig
class DBTLedgerModule extends DBTModule {
@Override
protected void configure() {
super.configure()
bind(LedgerChainAction)
bind(LedgerGetTransactionHandler)
bind(LedgerListTransactionsHandler)
bind(LedgerUpdateTransactionHandler)
bind(LedgerService)
Multibinder.newSetBinder(binder(), HandlerDecorator).addBinding()
.toInstance(HandlerDecorator.prependHandlers(LedgerChainAction))
}
@Override
protected void defaultConfig(ServerConfig serverConfig, DBTConfig config) {
config.ledger.enabled = !config.ledger.remoteUrl
if(!config.ledger.remoteUrl) {
config.ledger.remoteUrl = "http://localhost:${serverConfig.port}/${config.ledger.path}"
}
}
}

51
dbt-core/src/main/groovy/com/devsoap/dbt/services/LedgerService.groovy
View File

@@ -15,35 +15,72 @@
*/
package com.devsoap.dbt.services
import com.devsoap.dbt.config.DBTConfig
import com.devsoap.dbt.data.BlockTransaction
import com.devsoap.dbt.data.LedgerData
import com.fasterxml.jackson.databind.ObjectMapper
import com.gmongo.GMongo
import com.mongodb.BasicDBObject
import com.mongodb.BasicDBObjectBuilder
import com.mongodb.DB
import com.mongodb.DBCollection
import com.mongodb.DBCursor
import com.mongodb.DBObject
import com.mongodb.MongoURI
import com.mongodb.util.JSON
import groovy.util.logging.Slf4j
import ratpack.exec.Promise
import ratpack.service.Service
import javax.inject.Inject
@Slf4j
class LedgerService implements Service {
private static final LedgerData data = new LedgerData()
private final String dbUrl
private DB db
private final ObjectMapper mapper
@Inject
LedgerService(DBTConfig config, ObjectMapper mapper) {
dbUrl = config.ledger.databaseUrl
this.mapper = mapper
}
Promise<Optional<BlockTransaction>> fetchTransaction(String transactionId) {
Promise.value(Optional.ofNullable(data.transactions.find {it.id == transactionId}))
BlockTransaction transaction = transactions.findOne(['id':transactionId])?.findAll { it.key != '_id' } as BlockTransaction
Promise.value(Optional.ofNullable(transaction))
}
Promise<List<BlockTransaction>> allTransactions() {
Promise.value(data.transactions)
def cursor = transactions.find()
log.info("Found ${cursor.size()} transactions")
Promise.value(cursor.collect {it.findAll { it.key != '_id' } as BlockTransaction})
}
Promise<String> newTransaction(BlockTransaction transaction) {
log.info("Adding new transaction $transaction.id")
data.transactions.add(transaction)
transactions << JSON.parse(mapper.writeValueAsString(transaction))
Promise.value(transaction.id)
}
Promise<String> updateTransaction(BlockTransaction transaction) {
log.info("Updating transaction $transaction.id")
data.transactions.removeAll {it.id == transaction.id}
data.transactions.add(transaction)
transactions.findAndModify(
['id':transaction.id] as BasicDBObject,
JSON.parse(mapper.writeValueAsString(transaction)) as DBObject
)
Promise.value(transaction.id)
}
private DBCollection getTransactions() {
database.getCollection('transactions')
}
private DB getDatabase() {
if(!db) {
db = new GMongo(new MongoURI(dbUrl)).getDB('dbt')
}
db
}
}