From 97e2ed6dbca9e4ac3d7ade6ac70b9f0b9edc53f4 Mon Sep 17 00:00:00 2001 From: Thomas Teo Date: Mon, 14 Jan 2019 10:01:43 +1100 Subject: [PATCH 01/10] Test CI for container --- .gitlab-ci.yml | 23 +++++++++++++++++++++++ Dockerfile | 10 ++++++++++ 2 files changed, 33 insertions(+) create mode 100644 .gitlab-ci.yml create mode 100644 Dockerfile diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 0000000..cec7aab --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,23 @@ +# When using dind, it's wise to use the overlayfs driver for +# improved performance. +variables: + GIT_DEPTH: "3" + GIT_SUBMODULE_STRATEGY: "recursive" + GIT_SSL_NO_VERIFY: "true" +# DOCKER_DRIVER: overlay +stages: + - docker-build + +docker-build: + image: docker:latest + services: + - docker:dind + tags: + - dind + stage: docker-build + script: + # Begin the Docker build + - docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN $CI_REGISTRY + - docker build -t "$CI_REGISTRY/$CI_PROJECT_PATH:latest" . + - docker push "$CI_REGISTRY/$CI_PROJECT_PATH:latest" + diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..6b9be8c --- /dev/null +++ b/Dockerfile @@ -0,0 +1,10 @@ +from acfr/ros-build:kinetic + +# Add in the scripts +ADD scripts/* /opt/bagdb + +# Add in the dependencies +RUN apt-get update && apt-get -y install python-utm python-psycopg2 python-shapely + +# Add the scripts to the path +ENV PATH=/opt/bagdb:$PATH -- GitLab From 25624f56be404360e3146c044220461dcab2102a Mon Sep 17 00:00:00 2001 From: Thomas Teo Date: Mon, 14 Jan 2019 10:09:03 +1100 Subject: [PATCH 02/10] Fix path reference in Dockerfile. --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 6b9be8c..de1c642 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,7 @@ from acfr/ros-build:kinetic # Add in the scripts -ADD scripts/* /opt/bagdb +ADD scripts/* /opt/bagdb/ # Add in the dependencies RUN apt-get update && apt-get -y install python-utm python-psycopg2 python-shapely -- GitLab From 2ab1fd0f1a710e539e50c0c970a76b9174ecdd7f Mon Sep 17 00:00:00 2001 From: Thomas Teo Date: Mon, 14 Jan 2019 10:18:35 +1100 Subject: [PATCH 03/10] Add the ACFR ITS repo to the build to satisfy python-utm --- Dockerfile | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index de1c642..95d3a7a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,7 +4,10 @@ from acfr/ros-build:kinetic ADD scripts/* /opt/bagdb/ # Add in the dependencies -RUN apt-get update && apt-get -y install python-utm python-psycopg2 python-shapely +RUN echo "deb http://acfr-ros-pro-1.srv.sydney.edu.au:4440/zio-dev xenial main" > /etc/apt/sources.list.d/zio.list && \ +apt-get update && \ +apt-get -y install vim nano python-utm python-psycopg2 python-shapely && \ +apt clean all # Add the scripts to the path ENV PATH=/opt/bagdb:$PATH -- GitLab From 345739f0a09a2c3ce0f9134aff46873ad347380c Mon Sep 17 00:00:00 2001 From: Thomas Teo Date: Mon, 14 Jan 2019 10:19:30 +1100 Subject: [PATCH 04/10] Don't initialise the node when we're importing bags (removes need for rosmaster). --- scripts/test-bagdb.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/scripts/test-bagdb.py b/scripts/test-bagdb.py index f36c44e..d0e70b9 100644 --- a/scripts/test-bagdb.py +++ b/scripts/test-bagdb.py @@ -7,8 +7,6 @@ import rosbag import datetime -rospy.init_node('bag_database', anonymous=True) - bagdb = BagDB(dbname="bag_database") bags = [rosbag.Bag('/home/stew/data/EV/2018-09-13_seymour_auto_bumper/2018-09-13-14-14-05_seymour_auto_bumper.bag'), -- GitLab From b9bc55458f39f016f54d47a9b3b1be41de73872b Mon Sep 17 00:00:00 2001 From: Thomas Teo Date: Mon, 14 Jan 2019 10:33:29 +1100 Subject: [PATCH 05/10] Add repo key --- Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 95d3a7a..14f4284 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,7 +4,8 @@ from acfr/ros-build:kinetic ADD scripts/* /opt/bagdb/ # Add in the dependencies -RUN echo "deb http://acfr-ros-pro-1.srv.sydney.edu.au:4440/zio-dev xenial main" > /etc/apt/sources.list.d/zio.list && \ +RUN wget -O - http://acfr-ros-pro-1.srv.sydney.edu.au:4440/acfr_its.gpg.key | apt-key add - && \ +echo "deb http://acfr-ros-pro-1.srv.sydney.edu.au:4440/zio-dev xenial main" > /etc/apt/sources.list.d/zio.list && \ apt-get update && \ apt-get -y install vim nano python-utm python-psycopg2 python-shapely && \ apt clean all -- GitLab From 7b28813d2e7bcaa129691618bd30dfa766ad61e0 Mon Sep 17 00:00:00 2001 From: Thomas Teo Date: Mon, 14 Jan 2019 11:32:58 +1100 Subject: [PATCH 06/10] Implement command line arguments for bag files and config file. (Config file not yet handled) --- scripts/test-bagdb.py | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/scripts/test-bagdb.py b/scripts/test-bagdb.py index d0e70b9..8252f84 100644 --- a/scripts/test-bagdb.py +++ b/scripts/test-bagdb.py @@ -1,16 +1,31 @@ from BagDB import BagDB +import sys import json import time import rospy import rosbag import datetime +import argparse + +parser = argparse.ArgumentParser(description='Imports ROSbags to the SWRI bag-database.') +parser.add_argument('bag_files', metavar='abc.bag', nargs='+', + help='one or more ROSBag files.') + +parser.add_argument('--config', dest='config_file', default='/root/.ros-bag-database/settings.yml', + help='A bag-database style config file.') + +args = parser.parse_args() + +# TODO: pass the config file name to BagDB(); write a function in BagDB() to handle it bagdb = BagDB(dbname="bag_database") -bags = [rosbag.Bag('/home/stew/data/EV/2018-09-13_seymour_auto_bumper/2018-09-13-14-14-05_seymour_auto_bumper.bag'), - rosbag.Bag('/home/stew/data/EV/2018-08-28_seymour_centre_auto/2018-08-28-14-57-44_seymour_centre_auto.bag')] +bags = [] +for file in args.bag_files: + print (file) + bags.append(rosbag.Bag(file)) topics = bags[0].get_type_and_topic_info()[1].keys() -- GitLab From 6f5ddc6a4ea8b52867896428bcc52d4d386a58b9 Mon Sep 17 00:00:00 2001 From: Thomas Teo Date: Mon, 14 Jan 2019 11:37:17 +1100 Subject: [PATCH 07/10] Remove debugging. --- scripts/test-bagdb.py | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/test-bagdb.py b/scripts/test-bagdb.py index 8252f84..f0b8340 100644 --- a/scripts/test-bagdb.py +++ b/scripts/test-bagdb.py @@ -24,7 +24,6 @@ bagdb = BagDB(dbname="bag_database") bags = [] for file in args.bag_files: - print (file) bags.append(rosbag.Bag(file)) topics = bags[0].get_type_and_topic_info()[1].keys() -- GitLab From ddc02447387e496baf00f69e7915dc6aef4514ab Mon Sep 17 00:00:00 2001 From: Thomas Teo Date: Mon, 14 Jan 2019 13:43:44 +1100 Subject: [PATCH 08/10] Sample configuration file like what would be provided by bag-database. --- scripts/settings.yml | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 scripts/settings.yml diff --git a/scripts/settings.yml b/scripts/settings.yml new file mode 100644 index 0000000..c4b79bc --- /dev/null +++ b/scripts/settings.yml @@ -0,0 +1,13 @@ +!com.github.swrirobotics.support.web.Configuration +driver: org.postgresql.Driver +gpsTopics: [] +imuTopics: [] +jdbcPassword: letmein +jdbcUrl: jdbc:postgresql://bagdb-postgres/bag_database_acfr +jdbcUsername: bag_database +metadataTopics: [] +odometryTopics: [] +removeOnDeletion: false +vehicleNameTopics: +- /platform_name + -- GitLab From 4bac43ff43d74193a2ea9a04c41054804e531bea Mon Sep 17 00:00:00 2001 From: Thomas Teo Date: Mon, 14 Jan 2019 13:44:52 +1100 Subject: [PATCH 09/10] Support config files (now required on the command line). --- scripts/BagDB.py | 17 +++++++++++++++-- scripts/test-bagdb.py | 6 ++---- 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/scripts/BagDB.py b/scripts/BagDB.py index d593a36..ad7a779 100644 --- a/scripts/BagDB.py +++ b/scripts/BagDB.py @@ -8,6 +8,8 @@ import datetime import simplekml import intervaltree +import re +import yaml import numpy as np from shapely.geometry import asLineString @@ -16,12 +18,23 @@ from mpl_toolkits.mplot3d import Axes3D class BagDB: - def __init__(self, dbname='bag_database_acfr'): + def __init__(self, config_file): # Try to connect + if config_file: + # Open the config file, get parameters + with open(config_file, 'r') as ymlfile: + # Skip the first line, as it contains a nonstandard yaml format that pyyaml doesn't like. + for i in range(1): + _ = ymlfile.readline() + cfg = yaml.safe_load(ymlfile) + username = cfg['jdbcUsername'] + password = cfg['jdbcPassword'] + host, dbname = re.match('jdbc:postgresql://(.*?)/(.*)', cfg['jdbcUrl']).groups() + try: print ("connecting to the database") self.conn = psycopg2.connect( - "dbname=" + dbname + " user='bag_database' password='letmein' host='localhost' port=5432") + "dbname=" + dbname + " user=" + username + " password=" + password + " host=" + host + " port=5432") except: print "Unable to connect to the database." diff --git a/scripts/test-bagdb.py b/scripts/test-bagdb.py index f0b8340..bacf3d6 100644 --- a/scripts/test-bagdb.py +++ b/scripts/test-bagdb.py @@ -13,14 +13,12 @@ parser = argparse.ArgumentParser(description='Imports ROSbags to the SWRI bag-da parser.add_argument('bag_files', metavar='abc.bag', nargs='+', help='one or more ROSBag files.') -parser.add_argument('--config', dest='config_file', default='/root/.ros-bag-database/settings.yml', +parser.add_argument('--config', dest='config_file', default='/root/.ros-bag-database/settings.yml', required=True, help='A bag-database style config file.') args = parser.parse_args() -# TODO: pass the config file name to BagDB(); write a function in BagDB() to handle it - -bagdb = BagDB(dbname="bag_database") +bagdb = BagDB(config_file=args.config_file) bags = [] for file in args.bag_files: -- GitLab From c26b2c94a20609f6c1e3940c8e3f42c30430b739 Mon Sep 17 00:00:00 2001 From: Thomas Teo Date: Wed, 16 Jan 2019 09:25:55 +1100 Subject: [PATCH 10/10] Support specifying the bagid to be updated on the command line. This also necesitates only allowing a single bag file per execution. --- scripts/test-bagdb.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/scripts/test-bagdb.py b/scripts/test-bagdb.py index bacf3d6..3d64fef 100644 --- a/scripts/test-bagdb.py +++ b/scripts/test-bagdb.py @@ -10,12 +10,15 @@ import datetime import argparse parser = argparse.ArgumentParser(description='Imports ROSbags to the SWRI bag-database.') -parser.add_argument('bag_files', metavar='abc.bag', nargs='+', - help='one or more ROSBag files.') +parser.add_argument('bag_files', metavar='abc.bag', + help='A ROSBag file.') parser.add_argument('--config', dest='config_file', default='/root/.ros-bag-database/settings.yml', required=True, help='A bag-database style config file.') +parser.add_argument('--bagid', dest='bag_id', required=True, + help='The ID of the bag to be imported against.') + args = parser.parse_args() bagdb = BagDB(config_file=args.config_file) @@ -103,7 +106,7 @@ for bag_count, bag in enumerate(bags, 1): # TODO: set a relationship to the most recent global position message to allow searching batch_query_count += 1 - bagdb.AddMessageData(topic, unique_message_counter, bag_count, message_time, 0, 0, msg._type, message_json) + bagdb.AddMessageData(topic, unique_message_counter, bag_count, message_time, args.bag_id, 0, msg._type, message_json) if batch_query_count > 20000: print (".") -- GitLab