--- title: "simplaceUtil" author: "Gunther Krauss" date: "`r Sys.setlocale('LC_ALL','English.utf8');format(Sys.Date(),format='%B %d, %Y')`" output: rmarkdown::html_vignette: number_sections: yes fig-width: 10 fig-height: 8 toc: yes df_print: kable rmarkdown::pdf_document: number_sections: yes df_print: kable toc: yes vignette: > %\VignetteIndexEntry{simplaceUtil} %\VignetteEncoding{UTF-8} %\VignetteEngine{knitr::rmarkdown} --- ```{r setup, include=FALSE, error=FALSE} # withSimplace <- FALSE # try({ # withSimplace <- !is.na(simplace::findFirstSimplaceInstallation()) # }) knitr::opts_chunk$set(echo = TRUE) knitr::opts_chunk$set(message = FALSE) knitr::opts_chunk$set(warning = FALSE) includeGraphic <- function(gr,name) { type <- ifelse(knitr::is_latex_output(),'pdf','png') fn <- knitr::fig_path(type,number=as.integer(runif(1,0,10000))) dir.create(dirname(fn),recursive = TRUE, showWarnings = FALSE) DiagrammeR::export_graph(gr, fn, width = 2000) if(!file.exists(fn)) { fn <- system.file("img",paste0(name,'.png'),package="simplaceUtil") } try(knitr::include_graphics(fn), silent=TRUE) } rawOutput <- function(txt,type="") { cat(paste0("```",type,"\n")) cat(paste0(trimws(txt),collapse="\n")) cat("\n```") } ``` # Exploring solutions ## Fetch information from solution ```{r} library(simplaceUtil) ``` We can read a solution xml file via `getSolutionFromFile` and get its content as an `xml_document` object. From the xml object we can get the components, user variables and links between components as dataframes. ```{r} solfile <- system.file("solution","Complete.sol.xml", package="simplaceUtil") sol <- getSolutionFromFile(solfile) components <- getComponents(sol) variables <- getUserVariables(sol) links <- getLinks(sol, components) ``` As a shorthand we can use `getElementsFromSolutionFile`, which returns the xml object as well as the dataframes in a list. ```{r} el <- getElementsFromSolutionFile(solfile) sol <- el$solution components <- el$components variables <- el$variables links <- el$links ``` The components dataframe contains not only sim components, but also interfaces, resources and outputs. ```{r, echo=FALSE} components$ref <- gsub('$', '\\$', components$ref, fixed = TRUE) ``` ```{r} components[c(1:3,30:31,42:44),c("id","type","subtype","nr")] ``` The links dataframe contains the links between resources, simcomponents, outputs, but also dependencies via key variables or via variables used in rules. Relationships between resources and transformers are included too. ```{r} links[c(1:2,23,300,313,322,325),] ``` User variables show the values as defined in the solution. Notice that user variables can be changed when running a solution in project mode. ```{r} variables[c(1:5,18:20),] ``` Filenames for interfaces may use placeholder variables like `_WORKDIR_` or user defined variables (e.g. `vFilename`) ```{r} head(components[,c("id","ref")]) ``` ```{r, echo=FALSE} components$ref <- gsub('\\$', '$', components$ref, fixed = TRUE) ``` With `replaceVariablesWithValues` we can replace the placeholder with the values from the user value dataframe. We can supply additional values for system variables like `_WORKDIR_`. ```{r} components$ref <- replaceVariablesWithValues( components$ref, variables, additional=c("_WORKDIR_"="~")) head(components[,c("id","ref")]) ``` ## Working with component and links dataframe Subsetting the components dataframe to get only normal components ```{r} ncomp <- components[components$type=="simcomponent" & components$subtype=="normal",] ncomp$id ``` Getting only links between normal components ```{r} nlinks <- links[links$from %in% ncomp$id & links$to %in% ncomp$id,] head(nlinks) ``` Counting the links between components ```{r} nlinks <- aggregate(name ~ from + to , data=nlinks, length) nlinks <- nlinks[order(-nlinks$name),] head(nlinks,10) ``` ## Get Metadata about solution `getSolutionInfoAsDataframe` gives information about the solutions content (simcomponents) as well as the solution itself (filename, modification date and possibly simulation experiment information derived from folder structure). ```{r} wd <- system.file(package="simplaceUtil") solfiles <- list.files(wd,pattern=".sol.xml",recursive=TRUE,full.names = TRUE) df <- do.call(rbind, lapply(solfiles,getSolutionInfoAsDataframe,workdir=wd)) df$file <- basename(df$file) df[df$type=="simcomponent",c("id","file")] ``` # Visualising solution structure A graph showing components and links can be created from a solution file: ```{r} graph <- solutionToGraph(solfile) ``` ```{r eval=FALSE} DiagrammeR::render_graph(graph) ``` ```{r echo=FALSE, out.width="100%"} includeGraphic(graph,"graph") ``` We can restrict the graph to specific component types and the neighborhood of selected components: ```{r} subgraph <- getNeighborhood(graph, c("SlimWater","SlimRoots"), distance=1, type="simcomponent") ``` ```{r eval=FALSE} DiagrammeR::render_graph(subgraph) ``` ```{r echo=FALSE, out.width="100%"} includeGraphic(subgraph,"subgraph") ``` # Modifying solutions ## Sensitivity analysis and calibration A solution can be modified on the fly to meet requirements for sensitivity analysis or calibration by - removing non-memory outputs for performance reasons - adding memory outputs with target variables - adding user variables for parameters that will be varied - replacing variables from resource by user variables ```{r} solfile <- system.file("solution","Yield.sol.xml", package="simplaceUtil") sol <- getSolutionFromFile(solfile) newsol <- sol |> removeNonMemoryOutputs() |> addMemoryOutput("Yields",frequence = "YEARLY") |> addOutputVariable("Yields","Year","CURRENT.YEAR","INT") |> addOutputVariable("Yields", "ClimateZone","vClimateZone","CHAR") |> addOutputVariable("Yields", "Yield","LintulBiomass.sWSO","DOUBLE","MAX") |> addUserVariable("vLUE",3.0,"DOUBLE") |> addUserVariable("vSLA",0.02,"DOUBLE") |> addUserVariable("vRGRL",0.009,"DOUBLE") |> replaceVariable("crop.LUE","vLUE") |> replaceVariable("crop.SLA","vSLA") |> replaceVariable("crop.RGRL","vRGRL") ``` Let's output the beginning of the solution as text: \footnotesize ```{r} soltext <- getTextFromSolution(newsol) ``` ```{r echo=FALSE, results='asis'} rawOutput(substr(soltext,1,1530)) ``` \normalsize Notice that the modifications are logged in the `description` section of the new solution. The solution can be saved as xml file: ```{r eval=FALSE} solution_modified <- "simulation/solution/YieldModified.sol.xml" writeSolutionToFile(newsol, solution_modified) ``` ## Changing order of SimComponents As the order of SimComponents can have a considerable impact, one can generate random orders of components and run the reordered solutions to analyse the impact. ```{r} solfile <- system.file("solution","Yield.sol.xml", package="simplaceUtil") sol <- getSolutionFromFile(solfile) cmp <- getComponents(sol) cmp[cmp$type=="simcomponent",c("id","subtype","nr")] ``` We want to swap the components 2 to 5 ```{r echo=FALSE} set.seed(1234) ``` ```{r} toswap <- 2:5 swapped <- sample(toswap, length(toswap)) swapped newsol <- swapComponents(sol,swapped) cmp <- getComponents(newsol) cmp[cmp$type=="simcomponent",c("id","subtype","nr")] ``` The order of the SimComponent is now different. ## Adding timing SimComponent There is a special SimComponent that records the processing time of SimComponents. To add timing to a solution, one has to * add an interface for the timing output * add the timing SimComponent and configure it to measure the time of all or only of distinct components * add the output for the timing information This can be done for every solution automatically via `addTimingSimComponent`. ```{r} solfile <- system.file("solution","Yield.sol.xml", package="simplaceUtil") sol <- getSolutionFromFile(solfile) newsol <- addTimingSimComponent(sol, componentlist=c("LintulBiomass","LintulPheno")) cmp <- getComponents(newsol) cmp[cmp$type=="simcomponent" | grepl("timing",cmp$id),c("id","type","subtype","nr")] ``` The new solution has now an additional SimComponent *AutomaticTiming* as well as an additional output together with it's interface. # Standard transformation and visualisation of outputs ## Transform output data With `parseDate` the standard Date column of Simplace output files is converted to Date. ```{r} outfile <- system.file("output","water.csv", package="simplaceUtil") water <- read.csv(outfile, header=TRUE, sep=";") water <- parseDate(water) ``` By default the simplace outputs are in wide format, e.g. each layer is in an own column. We can transform the dataframe from wide format to long format, by putting values from all layers into one column and distinguish them by an extra column `layer`. ```{r} water[1:4,c(2,9:11,49)] water_long <- transformLayeredData(water) water_long[c(1:3,41:43,81:83),c(2,9:11)] ``` ## Visualise output variables Multiple scalar values can be plotted in a common diagram. ```{r fig.width=7, fig.height=5} plotScalarOutput(water,"CURRENT.DATE",c("Evaporation","Transpiration")) ``` Default plot for layered values. ```{r fig.width=7, fig.height=5} plotLayeredOutput(water,"RetainedWater") ``` # Simple GUI as Shiny App The package includes a Shiny App that runs locally within a webbrowser. ```{r eval=FALSE} runSimplaceGuiApp() ``` We can interactively explore solutions by filtering and displaying solution graphs. ```{r out.width="100%", echo=FALSE} fn <- system.file("img","shiny_graph.png",package="simplaceUtil") try(knitr::include_graphics(fn), silent=TRUE) ``` We can also use a local Simplace installation to run the simulation defined by the solution and plot some of the output variables. ```{r out.width="100%", echo=FALSE} fn <- system.file("img","shiny_plot.png",package="simplaceUtil") try(knitr::include_graphics(fn), silent=TRUE) ``` # Assistance for creating new Solutions (simulation configuration) ## Creating XML stubs for resources We can create the interface (location and format) and resource (structure) description for a solution as XML stubs from given input data. Data can be given in CSV or XML format. The structure of the data has to be compatible with Simplace. ```{r} stubs <- createResourceStubsFromCsv( filename = system.file("input","weather.csv", package="simplaceUtil"), id = "weather", sep =",", keyvals = c("CURRENT.DATE" = "Date") ) cat(stubs$interface) cat(stubs$resource) ``` The function tries to determine the datatype from the data. We have to check and possibly correct the inferred datatype (e.g. replacing 'CHAR' by 'DATE' or 'INT' by 'DOUBLE' or 'BOOLEAN'). # Assistance for developing new SimComponents ## Creating code stubs for SimVariables When creating a new SimComponent, we can put all SimVariables (constants, inputs, states, rates, outputs) in a CSV table. ```{r} varfile <- system.file("variables","translocation.csv",package="simplaceUtil") vartable <- read.csv(varfile, header=TRUE, sep=";") vartable[c(4,8:9),] ``` With the function `createCodeStubsForSimVariables` we can generate so called *boilerplate code* for Java as well as XML stubs for solutions. ```{r} stubs <- createCodeStubsForSimVariables(varfile, "BMTransloc", sep=";") names(stubs) ``` When we specify an output folder, the stubs are written to files. ```{r eval=FALSE} stubs <- createCodeStubsForSimVariables(varfile, "BMTransloc", sep=";", outfolder="translocation_stubs") ``` ## Example code stubs Defining fields in Java (`stubs$JavaFields`): ```{r echo=FALSE, results='asis'} rawOutput(stubs$JavaFields[c(4,6,14)]) ``` Creating the SimVariables (`stubs$JavaVariables`): ```{r echo=FALSE, results='asis'} rawOutput(stubs$JavaVariables[c(4,6,14)]) ``` Default initialisation of states and rates (sic!) (`stubs$JavaInit`): ```{r echo=FALSE, results='asis'} rawOutput(stubs$JavaInit[c(6,14)]) ``` XML stub for linking inputs (`stubs$ComponentInputsXML`): ```{r echo=FALSE, results='asis'} rawOutput(stubs$ComponentInputsXML[c(3:5)]) ``` XML stubs for parameter file (`stubs$ParametersXML`) and coresponding resource section in solution (`stubs$ResourceParametersXML`): ```{r echo=FALSE, results='asis'} rawOutput(stubs$ParametersXML) ``` ```{r echo=FALSE, results='asis'} rawOutput(stubs$ResourceParametersXML) ``` # Accessing SimComponent documentation With `fetchSimComponentlistFromWebsite()` one can get the java class names of the SimComponents from Simplace's website. With the class name one can then fetch the SimVariables as a dataframe ```{r} fetchSimVariablesFromWebsite('net.simplace.sim.components.evapotran.fao56.ReferenceETHargreaves')[,1:8] ```