code cleanup & add logging

opensrp · Feb 4, 2022 · 3ce5eac · 3ce5eac
1 parent cdf37a0
commit 3ce5eac
Show file tree

Hide file tree

Showing 9 changed files with 82 additions and 32 deletions.
diff --git a/README.md b/README.md
@@ -1,4 +1,4 @@
-# Opensrp DHIS2 Data Transfer
+# OpenSRP DHIS2 Data Transfer
 
 ## Build from source
 ```shell
@@ -13,15 +13,19 @@ java -jar opensrp-dhis2-data-transfer-0.0.1-SNAPSHOT.jar -export i -ds <DataSetN
 ```
 - Import organization units levels as location tags to opensrp server.
 ```shell
-java -jar opensrp-dhis2-data-transfer-0.0.1-SNAPSHOT.jar -export t
+java -jar opensrp-dhis2-data-transfer-0.0.1-SNAPSHOT.jar -import t
 ```
 - Import organization units as locations to opensrp server.
 ```shell
-java -jar opensrp-dhis2-data-transfer-0.0.1-SNAPSHOT.jar -export l
+java -jar opensrp-dhis2-data-transfer-0.0.1-SNAPSHOT.jar -import l
+```
+- Import organization units and organization unit levels to opensrp server.
+```shell
+java -jar opensrp-dhis2-data-transfer-0.0.1-SNAPSHOT.jar -import lt
 ```
 
 ## Configurations
-Configurations are store in `src/main/resources/application.properties`, update to your match your environment.
+Configurations are store in `src/main/resources/application.properties` or `application.properties` (has more precedence), update to your match your environment.
 
 | Configuration                  | Description                                              | Type    | Default               |
 |--------------------------------|----------------------------------------------------------|---------|-----------------------|

diff --git a/application.properties b/application.properties
@@ -0,0 +1,13 @@
+opensrp.url=http://localhost:8080
+opensrp.username=demo
+opensrp.password=password
+opensrp.keycloak.url=https://keycloak-url
+opensrp.keycloak.realm=realm
+opensrp.keycloak.client.id=client-id
+opensrp.keycloak.client.secret=client-secret
+dhis2.url=https://dhis2-url
+dhis2.username=user
+dhis2.password=district
+indicator.csv.name=indicators.csv
+location.tag.suffix=
+location.tag.getIdByName=true
diff --git a/build.gradle b/build.gradle
@@ -2,7 +2,7 @@ plugins {
     id 'org.jetbrains.kotlin.jvm' version '1.6.0'
 }
 
-group 'com.opensrp-dhis2-data-transfer'
+group 'org.smartregister.dhis2-data-transfer-tool'
 version '0.0.1-SNAPSHOT'
 
 repositories {

diff --git a/src/main/kotlin/ApplicationProperty.kt b/src/main/kotlin/ApplicationProperty.kt
@@ -1,10 +1,18 @@
+import java.io.File
+import java.io.FileInputStream
 import java.util.*
 
 object ApplicationProperty {
 
     private const val PROPERTY_FILE_NAME: String = "application.properties"
 
-    private val properties: Properties = Properties().apply { load(ClassLoader.getSystemClassLoader().getResourceAsStream(PROPERTY_FILE_NAME)) };
+    private val properties: Properties = Properties().apply {
+        val file = File(PROPERTY_FILE_NAME)
+        if(file.exists())
+            load(FileInputStream(file))
+        else
+            load(ClassLoader.getSystemClassLoader().getResourceAsStream(PROPERTY_FILE_NAME))
+    }
 
     fun getProperty(property: String): Any? {
         return properties.getOrDefault(property, null)

diff --git a/src/main/kotlin/configuration/Application.kt b/src/main/kotlin/configuration/Application.kt
@@ -4,10 +4,12 @@ import AppConstants
 import ApplicationProperty
 import com.fasterxml.jackson.databind.DeserializationFeature
 import com.fasterxml.jackson.dataformat.csv.CsvMapper
+import com.fasterxml.jackson.module.kotlin.KotlinFeature
 import com.fasterxml.jackson.module.kotlin.KotlinModule
+import com.fasterxml.jackson.module.kotlin.SingletonSupport
 import helper.IndicatorCsvHelper
 import helper.LocationImportHelper
-import helper.OpenrpAuthHelper.getOpensrpAuthToken
+import helper.OpensrpAuthHelper.getOpensrpAuthToken
 import io.ktor.client.*
 import io.ktor.client.engine.cio.*
 import io.ktor.client.features.*
@@ -21,13 +23,18 @@ import org.slf4j.LoggerFactory
 
 class Application {
 
-    object ApplicationLogger {
-        val logger = LoggerFactory.getLogger(javaClass)
-    }
-
     object ApplicationCsvMapper {
         val csvMapper = CsvMapper().apply {
-            registerModules(KotlinModule())
+            registerModules(
+                KotlinModule.Builder()
+                    .withReflectionCacheSize(512)
+                    .configure(KotlinFeature.NullToEmptyCollection, false)
+                    .configure(KotlinFeature.NullToEmptyMap, false)
+                    .configure(KotlinFeature.NullIsSameAsDefault, false)
+                    .configure(KotlinFeature.SingletonSupport, false)
+                    .configure(KotlinFeature.StrictNullChecks, false)
+                    .build()
+            )
         }
     }
 
@@ -73,38 +80,40 @@ class Application {
     }
 
     companion object {
+        val logger = LoggerFactory.getLogger(javaClass)
+
         @JvmStatic
         fun main(args: Array<String>) {
             if (args.isEmpty()) {
-                ApplicationLogger.logger.error("Command Line Arguments missing")
+                logger.error("Command Line Arguments missing")
                 return
             } else {
 
                 runBlocking {
                     launch {
                         if (args.size < 2) {
-                            ApplicationLogger.logger.error("Command Line Arguments missing")
+                            logger.error("Command Line Arguments missing")
                             return@launch
                         }
                         if (args.contains(AppConstants.Args.IMPORT) && ((args[args.indexOf(AppConstants.Args.IMPORT) + 1] == "tl") || (args[args.indexOf(
                                 AppConstants.Args.IMPORT
                             ) + 1] == AppConstants.Args.LOCATION_AND_TAG))
                         ) {
-                            ApplicationLogger.logger.info("Importing Locations and Location tags")
+                            logger.info("Importing Locations and Location tags")
                             LocationImportHelper().loadLocationTags()
                             LocationImportHelper().loadLocations()
                         } else if (args.contains(AppConstants.Args.IMPORT) && (args[args.indexOf(AppConstants.Args.IMPORT) + 1] == AppConstants.Args.LOCATION)) {
-                            ApplicationLogger.logger.info("Importing Locations")
+                            logger.info("Importing Locations")
                             LocationImportHelper().loadLocations()
                         } else if (args.contains(AppConstants.Args.IMPORT) && (args[args.indexOf(AppConstants.Args.IMPORT) + 1] == AppConstants.Args.LOCATION_TAG)) {
-                            ApplicationLogger.logger.info("Importing Location tags")
+                            logger.info("Importing Location tags")
                             LocationImportHelper().loadLocationTags()
                         } else if (args.size > 2 && args.contains(AppConstants.Args.EXPORT) && (args[args.indexOf(
                                 AppConstants.Args.EXPORT
                             ) + 1] == AppConstants.Args.INDICATOR)
                         ) {
                             if (args.contains(AppConstants.Args.DATASET) && (args[args.indexOf(AppConstants.Args.DATASET) + 1] != "")) {
-                                ApplicationLogger.logger.info("Exporting indicators")
+                                logger.info("Exporting indicators")
                                 val dataSets = args[args.indexOf(AppConstants.Args.DATASET) + 1]
                                 val indicatorCsvHelper = IndicatorCsvHelper()
                                 val dataSetArr = dataSets.split(",")
@@ -113,14 +122,14 @@ class Application {
                                     if (dataSetObject != null) {
                                         indicatorCsvHelper.fetchIndicators(dataSetObject)
                                     } else {
-                                        ApplicationLogger.logger.error("DataSet: $dataSet not found")
+                                        logger.error("DataSet: $dataSet not found")
                                     }
                                 }
                             } else {
-                                ApplicationLogger.logger.error("Arguments mismatch $args")
+                                logger.error("Arguments mismatch $args")
                             }
                         } else {
-                            ApplicationLogger.logger.error("Arguments mismatch $args")
+                            logger.error("Arguments mismatch $args")
                         }
                     }
                 }

diff --git a/src/main/kotlin/helper/IndicatorCsvHelper.kt b/src/main/kotlin/helper/IndicatorCsvHelper.kt
@@ -7,12 +7,15 @@ import io.ktor.client.call.*
 import io.ktor.client.request.*
 import io.ktor.client.statement.*
 import model.DataSet
+import org.slf4j.LoggerFactory
 import pojo.IndicatorCsv
 import utils.Utils
 import wrapper.DataSetWrapper
 
 open class IndicatorCsvHelper {
 
+    val logger = LoggerFactory.getLogger(javaClass)
+
     suspend fun getDataSetDetails(dataSetName: String): DataSet? {
         val client = Application.AppHttpClient.client
         val dhis2Url = ApplicationProperty.getProperty(AppConstants.Dhis2.URL)
@@ -66,13 +69,12 @@ open class IndicatorCsvHelper {
         val indicators: MutableList<IndicatorCsv> = mutableListOf()
 
         if (dataSetElements != null) {
+            logger.info("Indicators found.")
             for (dataSetElement in dataSetElements) {
                 val dataElement = dataSetElement.dataElement ?: return
                 val dataElementName = dataElement.name.trim()
                 val dataElementId = dataElement.id.trim()
                 val categoryCombo = dataElement.categoryCombo
-                val categoryComboName = categoryCombo.name.trim()
-                val categoryComboId = categoryCombo.id
                 if (categoryCombo.categoryOptionCombos != null) {
                     for (categoryOptionCombo in categoryCombo.categoryOptionCombos) {
                         val categoryComboOptionName = categoryOptionCombo.name.trim()
@@ -89,10 +91,13 @@ open class IndicatorCsvHelper {
                     }
                 }
             }
+        } else {
+            logger.info("No Indicators found. Kindly check the dataSet name")
         }
 
         val indicatorCsvFileName: String =
             ApplicationProperty.getProperty(AppConstants.Dhis2.INDICATOR_FILE_NAME) as String
+
         Utils.writeToCsv(indicators, "${dataSet.name.replace(" ", "_")}-$indicatorCsvFileName")
     }
 }
diff --git a/src/main/kotlin/helper/LocationImportHelper.kt b/src/main/kotlin/helper/LocationImportHelper.kt
@@ -9,6 +9,7 @@ import io.ktor.client.statement.*
 import io.ktor.http.*
 import io.ktor.util.*
 import model.*
+import org.slf4j.LoggerFactory
 import utils.Utils
 import utils.Utils.idToUUID
 import wrapper.OrganisationUnitLevelWrapper
@@ -17,6 +18,8 @@ import kotlin.system.exitProcess
 
 open class LocationImportHelper {
 
+    val logger = LoggerFactory.getLogger(javaClass)
+
     private suspend fun fetchOrganisationUnitLevels(): HashMap<Int, OrganisationUnitLevel> {
         val organisationUnitLevelMap: HashMap<Int, OrganisationUnitLevel> = HashMap()
         val client = Application.AppHttpClient.client
@@ -59,7 +62,7 @@ open class LocationImportHelper {
         if (!getLocationTagIdByName) {
             for (organisationUnitLevel in organisationUnitLevelMap.values) {
                 if (locationTagNameMap.containsKey(organisationUnitLevel.name)) {
-                    Application.ApplicationLogger.logger.error("LocationTag with the same name exists ${organisationUnitLevel.name}, Kindly fix before import begins")
+                    logger.error("LocationTag with the same name exists ${organisationUnitLevel.name}, Kindly fix before import begins")
                     exitProcess(1)
                 }
             }
@@ -69,9 +72,10 @@ open class LocationImportHelper {
                 val locationTag = LocationTag(0, organisationUnitLevel.name, true, organisationUnitLevel.name)
                 locationTags.add(locationTag)
 
-
+                var mappedByNameFromOpensrp = false;
                 if (getLocationTagIdByName && locationTagNameMap.containsKey(organisationUnitLevel.name)) {
                     locationTag.id = locationTagNameMap[organisationUnitLevel.name]?.id
+                    mappedByNameFromOpensrp = true;
                 }
 
                 val response: HttpResponse = client.put(url) {
@@ -82,12 +86,15 @@ open class LocationImportHelper {
                         append("Content-Type", "application/json")
                     }
                 }
+
+                val mapped = if(mappedByNameFromOpensrp) "mapped" else ""
+
                 if (response.status == HttpStatusCode.Created || response.status == HttpStatusCode.Accepted)
-                    Application.ApplicationLogger.logger.info("Loaded ${organisationUnitLevel.name} LocationTag")
+                    logger.info("Loaded ${organisationUnitLevel.name} LocationTag $mapped")
                 else
-                    Application.ApplicationLogger.logger.error("Problem occurred when loading ${organisationUnitLevel.name} locationTag")
+                    logger.error("Problem occurred when loading ${organisationUnitLevel.name} locationTag")
             } catch (e: Exception) {
-                Application.ApplicationLogger.logger.error(e)
+                logger.error(e)
             }
         }
         Utils.writeToCsv(locationTags, "locationTags.csv")
@@ -149,7 +156,7 @@ open class LocationImportHelper {
             }
             levelCount++
         }
-        Application.ApplicationLogger.logger.info("Loaded Locations")
+        logger.info("Loaded Locations")
     }
 
     private suspend fun populateLocations(
@@ -204,9 +211,9 @@ open class LocationImportHelper {
         }
 
         if (response.status == HttpStatusCode.Created || response.status == HttpStatusCode.Accepted)
-            Application.ApplicationLogger.logger.info("Loaded Locations ${pager?.page}/${pager?.pageCount} , Level:$levelCount/${organisationUnitLevelMap.size}")
+            logger.info("Loaded Locations ${pager?.page}/${pager?.pageCount} , Level:$levelCount/${organisationUnitLevelMap.size}")
         else
-            Application.ApplicationLogger.logger.error("Problem occurred when loading locations ${pager?.page}/${pager?.pageCount} , Level:$levelCount/${organisationUnitLevelMap.size}")
+            logger.error("Problem occurred when loading locations ${pager?.page}/${pager?.pageCount} , Level:$levelCount/${organisationUnitLevelMap.size}")
 
         return locations
     }

diff --git a/src/main/kotlin/helper/OpenrpAuthHelper.kt → src/main/kotlin/helper/OpensrpAuthHelper.kt b/src/main/kotlin/helper/OpenrpAuthHelper.kt → src/main/kotlin/helper/OpensrpAuthHelper.kt
@@ -11,7 +11,7 @@ import io.ktor.http.*
 import model.AuthToken
 import org.apache.commons.codec.binary.Base64
 
-object OpenrpAuthHelper {
+object OpensrpAuthHelper {
 
     suspend fun getOpensrpAuthToken(): AuthToken? {
         val client = Application.AppHttpClient.client

diff --git a/src/main/kotlin/utils/Utils.kt b/src/main/kotlin/utils/Utils.kt
@@ -1,15 +1,19 @@
 package utils
 
 import configuration.Application.ApplicationCsvMapper.csvMapper
+import org.slf4j.LoggerFactory
 import java.io.File
 import java.util.*
 
 
 object Utils {
+    val logger = LoggerFactory.getLogger(javaClass)
 
     inline fun <reified T> writeToCsv(data: Collection<T>, fileName: String) {
         csvMapper.writer(csvMapper.schemaFor(T::class.java).withHeader())
                 .writeValue(File(fileName), data)
+
+        logger.info("$fileName created.")
     }
 
     fun idToUUID(id: String?): String {