Commit 2ac0c66d authored by kaghog's avatar kaghog
Browse files

update the matsim setup and include specification for detailed activities in...

update the matsim setup and include specification for detailed activities in the matsim config creation
parent d325647c
......@@ -3,44 +3,80 @@ import shutil
def configure(context):
context.stage("matsim.run")
context.stage("matsim.simulation.run")
context.stage("matsim.simulation.prepare")
context.stage("matsim.runtime.eqasim")
context.config("output_path")
context.config("output_prefix", "switzerland_")
context.config("write_jar", True)
context.config("output_id")
context.stage("contracts.contracts")
context.stage("population.output")
def execute(context):
results_path = context.stage("matsim.run")
output_path = context.config("output_path")
output_id = context.config("output_id")
if not os.path.isdir(output_path):
raise RuntimeError("Output path does not exist:", output_path)
target_path = "%s/%s" % (output_path, output_id)
context.stage("population.output")
if os.path.exists(target_path):
if os.path.isdir(target_path):
print("Cleaning target directory:", target_path)
shutil.rmtree(target_path)
else:
raise RuntimeError("Cannot clean target path:", target_path)
#context.stage("documentation.meta_output")
context.stage("contracts.contracts")
os.mkdir(target_path)
def execute(context):
config_path = "%s/%s" % (
context.path("matsim.simulation.prepare"),
context.stage("matsim.simulation.prepare")
)
for file in [
"switzerland_network.xml.gz",
"switzerland_transit_schedule.xml.gz",
"switzerland_transit_vehicles.xml.gz",
"switzerland_facilities.xml.gz",
"switzerland_households.xml.gz",
"switzerland_population.xml.gz",
"switzerland_config.xml"
for name in [
"%shouseholds.xml.gz" % context.config("output_prefix"),
"%spopulation.xml.gz" % context.config("output_prefix"),
"%sfacilities.xml.gz" % context.config("output_prefix"),
"%snetwork.xml.gz" % context.config("output_prefix"),
"%stransit_schedule.xml.gz" % context.config("output_prefix"),
"%stransit_vehicles.xml.gz" % context.config("output_prefix"),
"%sconfig.xml" % context.config("output_prefix")
]:
shutil.copyfile("%s/%s" % (results_path, file), "%s/%s" % (target_path, file))
shutil.copy(
"%s/%s" % (context.path("matsim.simulation.prepare"), name),
"%s/%s" % (context.config("output_path"), name)
)
contracts_path = context.stage("contracts.contracts")
shutil.copyfile(contracts_path, "%s/CONTRACTS.html" % target_path)
if context.config("write_jar"):
shutil.copy(
"%s/%s" % (context.path("matsim.runtime.eqasim"), context.stage("matsim.runtime.eqasim")),
"%s/%srun.jar" % (context.config("output_path"), context.config("output_prefix"))
)
contracts_path = context.stage("contracts.contracts")
shutil.copyfile(contracts_path, "%s/CONTRACTS.html" % (context.config("output_path"), context.config("output_prefix")))
return {}
# def execute(context):
# results_path = context.stage("matsim.run")
#
# output_path = context.config("output_path")
# output_id = context.config("output_id")
#
# if not os.path.isdir(output_path):
# raise RuntimeError("Output path does not exist:", output_path)
#
# target_path = "%s/%s" % (output_path, output_id)
#
# if os.path.exists(target_path):
# if os.path.isdir(target_path):
# print("Cleaning target directory:", target_path)
# shutil.rmtree(target_path)
# else:
# raise RuntimeError("Cannot clean target path:", target_path)
#
# os.mkdir(target_path)
#
# for file in [
# "switzerland_network.xml.gz",
# "switzerland_transit_schedule.xml.gz",
# "switzerland_transit_vehicles.xml.gz",
# "switzerland_facilities.xml.gz",
# "switzerland_households.xml.gz",
# "switzerland_population.xml.gz",
# "switzerland_config.xml"
# ]:
# shutil.copyfile("%s/%s" % (results_path, file), "%s/%s" % (target_path, file))
#
# contracts_path = context.stage("contracts.contracts")
# shutil.copyfile(contracts_path, "%s/CONTRACTS.html" % target_path)
#
# return {}
import subprocess as sp
def configure(context):
context.stage("utils.java")
def execute(context):
java = context.stage("utils.java")
sp.check_call([
"git", "clone", "https://github.com/eqasim-org/eqasim-java.git"
], cwd=context.cache_path)
sp.check_call([
"git", "checkout", "v1.2.1"
], cwd="%s/eqasim-java" % context.cache_path)
sp.check_call([
"mvn", "-Pstandalone", "package"
], cwd="%s/eqasim-java" % context.cache_path)
jar = "%s/eqasim-java/switzerland/target/switzerland-1.2.1.jar" % context.cache_path
return jar
import os
import subprocess as sp
def configure(context):
context.stage("utils.java")
def execute(context):
java = context.stage("utils.java")
os.mkdir("%s/java_tmp" % context.cache_path)
sp.check_call([
"git", "clone", "https://github.com/matsim-org/pt2matsim.git"
], cwd=context.cache_path)
sp.check_call([
"git", "checkout", "v19.10"
], cwd="%s/pt2matsim" % context.cache_path)
sp.check_call([
"mvn", "-Djava.io.tmpdir=%s/java_tmp" % context.cache_path, "package"
], cwd="%s/pt2matsim" % context.cache_path)
jar = "%s/pt2matsim/target/pt2matsim-19.10-shaded.jar" % context.cache_path
java(jar, "org.matsim.pt2matsim.run.CreateDefaultOsmConfig", ["test_config.xml"], cwd=context.cache_path)
assert (os.path.exists("%s/test_config.xml" % context.cache_path))
assert (os.path.exists("%s/java_tmp/GeoTools" % context.cache_path))
return jar, "%s/java_tmp" % context.cache_path
......@@ -110,7 +110,7 @@ def execute(context):
person_writer.write(writer)
number_of_written_persons += 1
progress.update()
#progress.update()
except StopIteration:
pass
......
import shutil
def configure(context):
context.stage("matsim.population")
context.stage("matsim.households")
context.stage("matsim.facilities")
context.stage("matsim.network.mapped")
context.stage("matsim.java.eqasim")
context.stage("utils.java")
def execute(context):
# Some files we just copy
transit_vehicles_path = context.stage("matsim.network.mapped")["vehicles"]
shutil.copyfile(transit_vehicles_path, "%s/switzerland_transit_vehicles.xml.gz" % context.cache_path)
households_path = context.stage("matsim.households")
shutil.copyfile(households_path, "%s/switzerland_households.xml.gz" % context.cache_path)
# Some files we send through the preparation script
transit_schedule_input_path = context.stage("matsim.network.mapped")["schedule"]
transit_schedule_output_path = "%s/switzerland_transit_schedule.xml.gz" % context.cache_path
network_input_path = context.stage("matsim.network.mapped")["network"]
network_output_path = "%s/switzerland_network.xml.gz" % context.cache_path
facilities_input_path = context.stage("matsim.facilities")
facilities_output_path = "%s/switzerland_facilities.xml.gz" % context.cache_path
population_input_path = context.stage("matsim.population")
population_prepared_path = "%s/prepared_population.xml.gz" % context.cache_path
population_output_path = "%s/switzerland_population.xml.gz" % context.cache_path
config_output_path = "%s/switzerland_config.xml" % context.cache_path
# Call preparation script
java = context.stage("utils.java")
java(
context.stage("matsim.java.eqasim"),
"org.eqasim.core.scenario.preparation.RunPreparation", [
"--input-facilities-path", facilities_input_path,
"--output-facilities-path", facilities_output_path,
"--input-population-path", population_input_path,
"--output-population-path", population_prepared_path,
"--input-network-path", network_input_path,
"--output-network-path", network_output_path,
"--threads", str(context.config("threads"))
], cwd=context.cache_path)
java(
context.stage("matsim.java.eqasim"),
"org.eqasim.core.scenario.config.RunGenerateConfig", [
"--output-path", config_output_path,
"--prefix", "switzerland_",
"--sample-size", str(context.config("input_downsampling")),
"--random-seed", str(1000),
"--threads", str(context.config("threads"))
], cwd=context.cache_path)
java(
context.stage("matsim.java.eqasim"),
"org.eqasim.switzerland.scenario.RunCalculateStopCategories", [
"--input-path", transit_schedule_input_path,
"--output-path", transit_schedule_output_path
], cwd=context.cache_path)
java(
context.stage("matsim.java.eqasim"),
"org.eqasim.core.scenario.routing.RunPopulationRouting", [
"--config-path", config_output_path,
"--output-path", population_output_path,
"--threads", str(context.config("threads")),
"--config:plans.inputPlansFile", population_prepared_path
], cwd=context.cache_path)
java(
context.stage("matsim.java.eqasim"),
"org.eqasim.core.scenario.validation.RunScenarioValidator", [
"--config-path", config_output_path
], cwd=context.cache_path)
java(
context.stage("matsim.java.eqasim"),
"org.eqasim.switzerland.scenario.RunAdaptConfig", [
"--input-path", config_output_path,
"--output-path", config_output_path
], cwd=context.cache_path)
java(
context.stage("matsim.java.eqasim"),
"org.eqasim.switzerland.RunSimulation", [
"--config-path", config_output_path,
"--config:controler.lastIteration", str(1),
"--config:controler.writeEventsInterval", str(1),
"--config:controler.writePlansInterval", str(1),
], cwd=context.cache_path)
return context.cache_path
import subprocess as sp
import os, os.path, shutil
import matsim.runtime.git as git
import matsim.runtime.java as java
import matsim.runtime.maven as maven
def configure(context):
context.stage("matsim.runtime.git")
context.stage("matsim.runtime.java")
context.stage("matsim.runtime.maven")
context.config("eqasim_version", "1.3.1")
context.config("eqasim_branch", "v1.3.1")
context.config("eqasim_repository", "https://github.com/eqasim-org/eqasim-java.git")
context.config("eqasim_path", "")
def run(context, command, arguments):
version = context.config("eqasim_version")
# Make sure there is a dependency
context.stage("matsim.runtime.eqasim")
jar_path = "%s/eqasim-java/switzerland/target/switzerland-%s.jar" % (
context.path("matsim.runtime.eqasim"), version
)
java.run(context, command, arguments, jar_path)
def execute(context):
version = context.config("eqasim_version")
print("THIS IS A CHECK!!!")
print(context.config("eqasim_repository"))
# Normal case: we clone eqasim
if context.config("eqasim_path") == "":
# Clone repository and checkout version
git.run(context, [
"clone", context.config("eqasim_repository"),
"--branch", context.config("eqasim_branch"),
"--single-branch", "eqasim-java",
"--depth", "1"
])
# Build eqasim
maven.run(context, ["-Pstandalone", "--projects", "switzerland", "--also-make", "package"], cwd = "%s/eqasim-java" % context.path())
jar_path = "%s/eqasim-java/switzerland/target/switzerland-%s.jar" % (context.path(), version)
# Special case: We provide the jar directly. This is mainly used for
# creating input to unit tests of the eqasim-java package.
else:
os.makedirs("%s/eqasim-java/switzerland/target" % context.path())
shutil.copy(context.config("eqasim_path"),
"%s/eqasim-java/switzerland/target/switzerland-%s.jar" % (context.path(), version))
return "eqasim-java/switzerland/target/switzerland-%s.jar" % version
def validate(context):
path = context.config("eqasim_path")
if path == "":
return True
if not os.path.exists(path):
raise RuntimeError("Cannot find eqasim at: %s" % path)
return os.path.getmtime(path)
\ No newline at end of file
import subprocess as sp
import shutil
def configure(context):
context.config("git_binary", "git")
def run(context, arguments = [], cwd = None, catch_output = False):
"""
This function calls git.
"""
# Make sure there is a dependency
context.stage("matsim.runtime.git")
if cwd is None:
cwd = context.path()
command_line = [
shutil.which(context.config("git_binary"))
] + arguments
if catch_output:
return sp.check_output(command_line, cwd = cwd).decode("utf-8").strip()
else:
return_code = sp.check_call(command_line, cwd = cwd)
if not return_code == 0:
raise RuntimeError("Git return code: %d" % return_code)
def validate(context):
if shutil.which(context.config("git_binary")) == "":
raise RuntimeError("Cannot find git binary at: %s" % context.config("git_binary"))
if not b"2." in sp.check_output([
shutil.which(context.config("git_binary")),
"--version"
], stderr = sp.STDOUT):
print("WARNING! Git of at least version 2.x.x is recommended!")
def execute(context):
pass
\ No newline at end of file
import subprocess as sp
import os, shutil
def configure(context):
context.config("java_binary", "java")
context.config("java_memory", "50G")
def run(context, entry_point, arguments = [], class_path = None, vm_arguments = [], cwd = None, memory = None, mode = "raise"):
"""
This function calls java code. There are three modes:
- return_code: Returns the return code of the Java call
- output: Returns the output of the Java call
- raise (default): Raises an exception if the return code is not zero
"""
# Make sure there is a dependency
context.stage("matsim.runtime.java")
# Prepare temp folder
temp_path = "%s/__java_temp" % context.path()
if not os.path.exists(temp_path):
os.mkdir(temp_path)
# Prepare arguments
memory = context.config("java_memory") if memory is None else memory
vm_arguments = [
"-Xmx" + memory,
"-Djava.io.tmpdir=%s" % temp_path,
"-Dmatsim.useLocalDtds=true"
] + vm_arguments
# Prepare classpath
if type(class_path) == list or type(class_path) == tuple:
class_path = ":".join(class_path)
# Prepare CWD
if cwd is None:
cwd = context.path()
# Prepare command line
command_line = [
shutil.which(context.config("java_binary")),
"-cp", class_path
] + vm_arguments + [
entry_point
] + arguments
command_line = list(map(str, command_line))
print("Executing java:", " ".join(command_line))
if mode == "raise" or mode == "return_code":
return_code = sp.check_call(command_line, cwd = cwd)
if not return_code == 0:
raise RuntimeError("Java return code: %d" % return_code)
return return_code
elif mode == "output":
return sp.check_output(command_line, cwd = cwd)
else:
raise RuntimeError("Mode is expected to be one of 'raise', 'return_code' or 'output'")
def validate(context):
if shutil.which(context.config("java_binary")) == "":
raise RuntimeError("Cannot find Java binary at: %s" % context.config("java_binary"))
if not b"11" in sp.check_output([
shutil.which(context.config("java_binary")),
"-version"
], stderr = sp.STDOUT):
print("WARNING! A Java JDK of at least version 11 is recommended.")
def execute(context):
pass
\ No newline at end of file
import subprocess as sp
import os, shutil
def configure(context):
context.config("maven_binary", "mvn")
context.config("maven_skip_tests", False)
def run(context, arguments = [], cwd = None):
"""
This function calls Maven.
"""
# Make sure there is a dependency
context.stage("matsim.runtime.maven")
if cwd is None:
cwd = context.path()
# Prepare temp folder
temp_path = "%s/__java_temp" % context.path()
if not os.path.exists(temp_path):
os.mkdir(temp_path)
vm_arguments = [
"-Djava.io.tmpdir=%s" % temp_path
]
if context.config("maven_skip_tests"):
vm_arguments.append("-DskipTests=true")
command_line = [
shutil.which(context.config("maven_binary"))
] + vm_arguments + arguments
return_code = sp.check_call(command_line, cwd = cwd)
if not return_code == 0:
raise RuntimeError("Maven return code: %d" % return_code)
def validate(context):
if shutil.which(context.config("maven_binary")) == "":
raise RuntimeError("Cannot find Maven binary at: %s" % context.config("maven_binary"))
if not b"3." in sp.check_output([
shutil.which(context.config("maven_binary")),
"-version"
], stderr = sp.STDOUT):
print("WARNING! Maven of at least version 3.x.x is recommended!")
def execute(context):
pass
\ No newline at end of file
import subprocess as sp
import os, os.path
import matsim.runtime.git as git
import matsim.runtime.java as java
import matsim.runtime.maven as maven
def configure(context):
context.stage("matsim.runtime.git")
context.stage("matsim.runtime.java")
context.stage("matsim.runtime.maven")
context.config("pt2matsim_version", "21.4")
context.config("pt2matsim_branch", "v21.4")
def run(context, command, arguments):
version = context.config("pt2matsim_version")
# Make sure there is a dependency
context.stage("matsim.runtime.pt2matsim")
jar_path = "%s/pt2matsim/target/pt2matsim-%s-shaded.jar" % (
context.path("matsim.runtime.pt2matsim"), version
)
java.run(context, command, arguments, jar_path)
def execute(context):
version = context.config("pt2matsim_version")
branch = context.config("pt2matsim_branch")
# Clone repository and checkout version
git.run(context, [
"clone", "https://github.com/matsim-org/pt2matsim.git",
"--branch", branch,
"--single-branch", "pt2matsim",
"--depth", "1"
])
# Build pt2matsim
maven.run(context, ["package"], cwd = "%s/pt2matsim" % context.path())
jar_path = "%s/pt2matsim/target/pt2matsim-%s-shaded.jar" % (context.path(), version)
# Test pt2matsim
java.run(context, "org.matsim.pt2matsim.run.CreateDefaultOsmConfig", [
"test_config.xml"
], jar_path)
assert os.path.exists("%s/test_config.xml" % context.path())
\ No newline at end of file
import gzip
import io
import io, gzip
import pandas as pd
import matsim.writers
import numpy as np
from synpp import progress
import matsim.writers as writers
def configure(context):
......@@ -69,7 +70,7 @@ def execute(context):
with gzip.open("%s/facilities.xml.gz" % cache_path, "w+") as f:
with io.BufferedWriter(f, buffer_size=1024 * 1024 * 1024 * 2) as raw_writer:
writer = matsim.writers.FacilitiesWriter(raw_writer)
writer = writers.FacilitiesWriter(raw_writer)
writer.start_facilities()
for item in context.progress(df_statent.itertuples(), total=len(df_statent)):
......@@ -107,7 +108,9 @@ def execute(context):
writer.start_facility("home%s" % item[1], item[2], item[3])
writer.add_activity("home")
writer.end_facility()
#progress.update()
writer.end_facilities()
return "%s/facilities.xml.gz" % cache_path
#return "%s/facilities.xml.gz" % cache_path
return "facilities.xml.gz"
......@@ -4,7 +4,7 @@ import io
import numpy as np
import data.constants as c
import matsim.writers
import matsim.writers as writers