Experiment

Experiment

Experiment

Value

an object of class Experiment

Details

Methods

control(...)

control code block

candidate(...)

candidate code block

run(...)

execute code, runs call_block(), then do_comparison() ...: pass on parameters through call_block() to callr::r() (wait=TRUE) or callr::r_bg() (wait=FALSE)

call_block()

execute code, and collect timing data

do_comparison()

mostly an internal fxn, run after call_block

result()

fetch the result (named list)

publish(browse = TRUE)

publish results. creates an html file. if browse=TRUE the file opens in your default browser. if browse=FALSE you get a file path.

diff()

compare results by "diffing" in the R console. used inside of publish() to prepare diffs. different diff representations for different object types:

• numeric/integer: take difference between 2, or matrix comparing > 2 results

• character/factor: highlight differences in strings, like git diffs

• data.frame/matrix: use visdat

• images: use vdiffr

• list: not sure how to do this

• environments: not sure how to do this

compare(x = NULL)

set a custom comparison function, must result in a single boolean fun: a function

Terminology

The control block is the control, the current state of the code. The candidate block is the candidate, or the new version of the code you want to compare to the control.

Methods

Public methods

• Experiment$new()

• Experiment$print()

• Experiment$call_block()

• Experiment$status()

• Experiment$collect()

• Experiment$control()

• Experiment$candidate()

• Experiment$run()

• Experiment$compare()

• Experiment$do_comparison()

• Experiment$result()

• Experiment$publish()

• Experiment$diff()

• Experiment$clone()

---

Method new()

Usage

Experiment$new(name, error_on_mismatch = FALSE, wait = TRUE, progress = FALSE)

Arguments

name

The name of the experiment

error_on_mismatch

(logical) whether to error on mismatch of results. default: FALSE

wait

(logical) wait for code to execute. if FALSE, code is run in the background, and you have to run $collect() to collect results. default: TRUE

progress

(logical) whether to turn on progress information or not, default: TRUE (IGNORED RIGHT NOW)

---

Method print()

Usage

Experiment$print(...)

---

Method call_block()

Usage

Experiment$call_block(...)

---

Method status()

Usage

Experiment$status()

---

Method collect()

Usage

Experiment$collect()

---

Method control()

Usage

Experiment$control(...)

---

Method candidate()

Usage

Experiment$candidate(...)

---

Method run()

Usage

Experiment$run(...)

---

Method compare()

Usage

Experiment$compare(x = NULL)

---

Method do_comparison()

Usage

Experiment$do_comparison()

---

Method result()

Usage

Experiment$result()

---

Method publish()

Usage

Experiment$publish(browse = TRUE)

---

Method diff()

Usage

Experiment$diff()

---

Method clone()

The objects of this class are cloneable with this method.

Usage

Experiment$clone(deep = FALSE)

Arguments

deep

Whether to make a deep clone.

Examples

library(scientist)

# basic eg
res <- Experiment$new(name = "jane")
res$control({
  x = 5
  x^2
})
res$candidate({
  y = 5
  y^3
})
res
#> <Experiment> jane
#>  error on mismatch?: FALSE
#>  waiting?: TRUE
#>  progress?: FALSE
#>   control: <unnamed>
#>   candidate: <unnamed>
res$run()
#> waiting ...
#> done ...
res$control_result
#> [[1]]
#> [1] 25
#> 
res$candidate_results
#> [[1]]
#> [1] 125
#> 
res$result()
#> $name
#> [1] "jane"
#> 
#> $control
#> $control$result
#> $control$result[[1]]
#> [1] 25
#> 
#> 
#> $control$time
#> $control$time$start
#> [1] "2020-09-18 01:12:55 GMT"
#> 
#> $control$time$end
#> [1] "2020-09-18 01:12:56 GMT"
#> 
#> $control$time$duration
#> [1] 0.647732
#> 
#> 
#> 
#> $candidates
#> $candidates[[1]]
#> $candidates[[1]]$result
#> [1] 125
#> 
#> $candidates[[1]]$time
#> $candidates[[1]]$time$start
#> [1] "2020-09-18 01:12:55 GMT"
#> 
#> $candidates[[1]]$time$end
#> [1] "2020-09-18 01:12:56 GMT"
#> 
#> $candidates[[1]]$time$duration
#> [1] 0.6126211
#> 
#> 
#> $candidates[[1]]$name
#> [1] NA
#> 
#> 
#> 
#> $comparison
#> [1] FALSE
#> 
# publish results
res$publish()

# many candidates
res <- Experiment$new(name = "doe")
res
#> <Experiment> doe
#>  error on mismatch?: FALSE
#>  waiting?: TRUE
#>  progress?: FALSE
#>   control: <not assigned>
#>   candidate: <not assigned>
res$control(stuff = {
  Sys.sleep(2)
  x = 5
  x^2
})
res$candidate(foo = {
  Sys.sleep(2)
  y = 5
  y^3
}, bar = {
  Sys.sleep(2)
  w = 1000
  (w - 20) / 34
})
res
#> <Experiment> doe
#>  error on mismatch?: FALSE
#>  waiting?: TRUE
#>  progress?: FALSE
#>   control: stuff
#>   candidate: foo
#>   candidate: bar
res$run()
#> waiting ...
#> done ...
res$control_result
#> $stuff
#> [1] 25
#> 
res$candidate_results
#> $foo
#> [1] 125
#> 
#> $bar
#> [1] 28.82353
#> 
res$result()
#> $name
#> [1] "doe"
#> 
#> $control
#> $control$result
#> $control$result$stuff
#> [1] 25
#> 
#> 
#> $control$time
#> $control$time$start
#> [1] "2020-09-18 01:12:56 GMT"
#> 
#> $control$time$end
#> [1] "2020-09-18 01:12:59 GMT"
#> 
#> $control$time$duration
#> [1] 3.26573
#> 
#> 
#> 
#> $candidates
#> $candidates$foo
#> $candidates$foo$result
#> [1] 125
#> 
#> $candidates$foo$time
#> $candidates$foo$time$start
#> [1] "2020-09-18 01:12:56 GMT"
#> 
#> $candidates$foo$time$end
#> [1] "2020-09-18 01:13:00 GMT"
#> 
#> $candidates$foo$time$duration
#> [1] 3.362396
#> 
#> 
#> $candidates$foo$name
#> [1] "foo"
#> 
#> 
#> $candidates$bar
#> $candidates$bar$result
#> [1] 28.82353
#> 
#> $candidates$bar$time
#> $candidates$bar$time$start
#> [1] "2020-09-18 01:12:57 GMT"
#> 
#> $candidates$bar$time$end
#> [1] "2020-09-18 01:13:00 GMT"
#> 
#> $candidates$bar$time$duration
#> [1] 3.147687
#> 
#> 
#> $candidates$bar$name
#> [1] "bar"
#> 
#> 
#> 
#> $comparison
#> [1] FALSE
#> 
res$times
#> $control_time
#> $control_time$start
#> [1] "2020-09-18 01:12:56 GMT"
#> 
#> $control_time$end
#> [1] "2020-09-18 01:12:59 GMT"
#> 
#> $control_time$duration
#> [1] 3.26573
#> 
#> 
#> $candidate_times
#> $candidate_times$foo
#> $candidate_times$foo$start
#> [1] "2020-09-18 01:12:56 GMT"
#> 
#> $candidate_times$foo$end
#> [1] "2020-09-18 01:13:00 GMT"
#> 
#> $candidate_times$foo$duration
#> [1] 3.362396
#> 
#> 
#> $candidate_times$bar
#> $candidate_times$bar$start
#> [1] "2020-09-18 01:12:57 GMT"
#> 
#> $candidate_times$bar$end
#> [1] "2020-09-18 01:13:00 GMT"
#> 
#> $candidate_times$bar$duration
#> [1] 3.147687
#> 
#> 
#> 
# publish results
res$publish()

# raise errors when the control and experiment do no match
res <- Experiment$new(name = "treetest", error_on_mismatch = TRUE)
res$control({
  x = 5
  x^2
})
res$candidate({
  y = 5
  y^3
})
res
#> <Experiment> treetest
#>  error on mismatch?: TRUE
#>  waiting?: TRUE
#>  progress?: FALSE
#>   control: <unnamed>
#>   candidate: <unnamed>
if (FALSE) res$run()

# if not waiting, run $status() and $collect()
res <- Experiment$new(name = "junipers", wait = FALSE)
res$control({
  x = 5
  x^2
})
res$candidate({
  y = 5
  y^3
})
if (FALSE) res$status()
res$run()
res
#> <Experiment> junipers
#>  error on mismatch?: FALSE
#>  waiting?: FALSE
#>  progress?: FALSE
#>   control: <unnamed>
#>   candidate: <unnamed>
res$status()
#> $control
#> [1] FALSE
#> 
#> $candidates
#> $candidates[[1]]
#> [1] FALSE
#> 
#> 
#> attr(,"class")
#> [1] "exp_status"
if (FALSE) res$collect()
res$result()
#> $name
#> [1] "junipers"
#> 
#> $control
#> $control$result
#> NULL
#> 
#> $control$time
#> NULL
#> 
#> 
#> $candidates
#> list()
#> 
#> $comparison
#> [1] TRUE
#> 

# set explicit comparison
# FIXME: not working yet
# res <- Experiment$new(name = "jane")
# res$control({
#   x = 5
#   x^2
# })
# res$candidate({
#   y = 5
#   y^2
# })
# res$compare(function(control, candidate) {
#   control/2 == candidate/1
# })
# res$run()
# res$result()