julialogging / tensorboardlogger.jl Goto Github PK

On windows:
I follow the histogram example in https://github.com/PhilipVinc/TensorBoardLogger.jl/blob/master/examples/Histograms.jl

using TensorBoardLogger #import the TensorBoardLogger package
using Logging #import Logging package

logger = TBLogger("histogramlogs", tb_append) #create tensorboard logger

################log histogram example:################
#using logger interface
with_logger(logger) do
    for i in 1:100
        x0 = 0.5+i/30; s0 = 0.5/(i/20);
        edges = collect(-5:0.1:5)
        centers = collect(edges[1:end-1] .+0.05)
        histvals = [exp(-((c-x0)/s0)^2) for c = centers]
        data_tuple = (edges, histvals)
        @info "histogram/loggerinterface" autobin=rand(10).+0.1*i manualbin=data_tuple
    end
end

Running it in tensorboard on my computer, everything looks great. i.e.

tensorboard --logdir histogramlogs

However, when I follow the instructions to upload it,

tensorboard dev upload --logdir \ 
     'histogramlogs'

It returns a link for tensorboard, but the output says

PermissionError: [WinError 5] Access is denied: '\\$Recycle.Bin\\S-1-5-18'

I tried running as an administrator and it didn't help, but the \\$Recyle.Bin is very suspicious.

The full output

S C:\Users\jlperla\Documents\GitHub\DifferentiableStateSpaceModels.jl> tensorboard dev upload --logdir \
TensorFlow installation not found - running with reduced feature set.
Data for the "graphs" plugin is now uploaded to TensorBoard.dev! Note that uploaded data is public. If you do not want to upload data for this plugin, use the "--plugins" command line argument.
Data for the "histograms" plugin is now uploaded to TensorBoard.dev! Note that uploaded data is public. If you do not want to upload data for this plugin, use the "--plugins" command line argument.
Data for the "hparams" plugin is now uploaded to TensorBoard.dev! Note that uploaded data is public. If you do not want to upload data for this plugin, use the "--plugins" command line argument.
Upload started and will continue reading any new data as it's added
to the logdir. To stop uploading, press Ctrl-C.

View your TensorBoard live at: https://tensorboard.dev/experiment/QWseahp3Rfy8MKLWoiVtKg/


Done. View your TensorBoard at https://tensorboard.dev/experiment/QWseahp3Rfy8MKLWoiVtKg/
Traceback (most recent call last):
  File "c:\users\jlperla\anaconda3\lib\runpy.py", line 193, in _run_module_as_main
    "__main__", mod_spec)
  File "c:\users\jlperla\anaconda3\lib\runpy.py", line 85, in _run_code
    exec(code, run_globals)
  File "C:\Users\jlperla\Anaconda3\Scripts\tensorboard.exe\__main__.py", line 9, in <module>
  File "c:\users\jlperla\anaconda3\lib\site-packages\tensorboard\main.py", line 75, in run_main
    app.run(tensorboard.main, flags_parser=tensorboard.configure)
  File "c:\users\jlperla\anaconda3\lib\site-packages\absl\app.py", line 300, in run
    _run_main(main, args)
  File "c:\users\jlperla\anaconda3\lib\site-packages\absl\app.py", line 251, in _run_main
    sys.exit(main(argv))
  File "c:\users\jlperla\anaconda3\lib\site-packages\tensorboard\program.py", line 290, in main
    return runner(self.flags) or 0
  File "c:\users\jlperla\anaconda3\lib\site-packages\tensorboard\uploader\uploader_subcommand.py", line 633, in run
    return _run(flags)
  File "c:\users\jlperla\anaconda3\lib\site-packages\tensorboard\uploader\uploader_subcommand.py", line 140, in _run
    intent.execute(server_info, channel)
  File "c:\users\jlperla\anaconda3\lib\site-packages\tensorboard\uploader\uploader_subcommand.py", line 447, in execute
    uploader.start_uploading()
  File "c:\users\jlperla\anaconda3\lib\site-packages\tensorboard\uploader\uploader.py", line 199, in start_uploading
    self._upload_once()
  File "c:\users\jlperla\anaconda3\lib\site-packages\tensorboard\uploader\uploader.py", line 213, in _upload_once
    self._logdir_loader.synchronize_runs()
  File "c:\users\jlperla\anaconda3\lib\site-packages\tensorboard\uploader\logdir_loader.py", line 70, in synchronize_runs
    for subdir in io_wrapper.GetLogdirSubdirectories(self._logdir):
  File "c:\users\jlperla\anaconda3\lib\site-packages\tensorboard\backend\event_processing\io_wrapper.py", line 232, in <genexpr>
    subdir
  File "c:\users\jlperla\anaconda3\lib\site-packages\tensorboard\backend\event_processing\io_wrapper.py", line 183, in ListRecursivelyViaWalking
    for dir_path, _, filenames in tf.io.gfile.walk(top, topdown=True):
  File "c:\users\jlperla\anaconda3\lib\site-packages\tensorboard\compat\tensorflow_stub\io\gfile.py", line 719, in walk
    for subitem in walk(joined_subdir, topdown, onerror=onerror):
  File "c:\users\jlperla\anaconda3\lib\site-packages\tensorboard\compat\tensorflow_stub\io\gfile.py", line 719, in walk
    for subitem in walk(joined_subdir, topdown, onerror=onerror):
  File "c:\users\jlperla\anaconda3\lib\site-packages\tensorboard\compat\tensorflow_stub\io\gfile.py", line 696, in walk
    listing = listdir(top)
  File "c:\users\jlperla\anaconda3\lib\site-packages\tensorboard\compat\tensorflow_stub\io\gfile.py", line 658, in listdir
    return get_filesystem(dirname).listdir(dirname)
  File "c:\users\jlperla\anaconda3\lib\site-packages\tensorboard\compat\tensorflow_stub\io\gfile.py", line 193, in listdir
    entries = os.listdir(compat.as_str_any(dirname))
PermissionError: [WinError 5] Access is denied: '\\$Recycle.Bin\\S-1-5-18'

Note the bottom,

When building a graph (#32), it is very important to obtain the name of the function that was executed to obtain each Tracked variable.
e.g.

x = param(rand(2,2))
y = param(rand(2,2))
z = x*y

I want to find the symbol * from z alone.

Something like the following should work.

It just needs testing and screenshots added.

using TensorBoardLogger
using Optim
using Logging

# Returns a closure over a logger, that takes a Optim trace as input
function make_tensorboardlogger_callback(dir "logs")
    logger = TensorBoardLogger(dir)

    function callback(opt_state = Optim.OptimizationState)
        withlogger(logger) do
            @info "" opt_step = opt_state.iteration  function_value=opt_state.value gradient_norm=opt_state.g_norm
        end
        return false  # do not terminate optimisation
    end 
    callback(trace::Optim.OptimizationTrace) = callback(last(trace))
    return callback
end


rosenbrock(x) =  (1.0 - x[1])^2 + 100.0 * (x[2] - x[1]^2)^2
result = optimize(rosenbrock, zeros(2), BFGS(); callback=make_tensorboardlogger_callback())

cc @pkofod

Unsure whether to implement Logger dispatch for SampleBuf. It will result in SampledSignals dependency.
https://github.com/JuliaAudio/SampledSignals.jl/blob/master/src/SampleBuf.jl
#27

Tensorboard supports logging hyperparameters (see here). It would be good if there were an option when creating a logger to specify the hyperparameters of the run so they they can be view in the HParams section of tensorboard.

Following #46 I realised that Optimums stores optimisation metadata in Dictionaries.

Right now, if a dictionary is logged to TBLogger (or if it's found while unpacking other structs) it is treated like any other struct, and it's internal fields are recursively logged:

Julia> using TensorBoardLogger
julia> data = TensorBoardLogger.preprocess("dict", Dict(:a=>1, :b=>2), [])
8-element Array{Any,1}:
    "dict/slots" => UInt8[0x01, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00]
     "dict/keys" => Symbol[:a, :b, #undef, #undef, #undef, #undef, #undef, #undef, #undef, #undef, #undef, #undef, #undef, #undef, #undef, #undef]
     "dict/vals" => [1, 2, 32, 1, 11, 2, 4628399872, 108, 24, 4628399872, 4628400112, 4628402464, 4628400112, 4628398656, 4628400112, 4426404768]
     "dict/ndel" => 0
    "dict/count" => 2
      "dict/age" => 0x0000000000000002
 "dict/idxfloor" => 1
 "dict/maxprobe" => 0

I think we should treat dictionaries with String and Symbol keys as we treat structures, and further preprocess key-value pairs.

Although, people often don't specialise the key type and use Dict{Any,Any} (Optim does that).
In this case, we could check if a string representation of every key exists, otherwise log them both as text?

MWE:

julia> using TensorBoardLogger, Logging, ValueHistories

julia> lg = TBLogger(joinpath(mktempdir(), "runs"), min_level=Logging.Info; step_increment=0)
TBLogger:
    - Log level     : Info
    - Current step  : 0
    - Output        : /tmp/jl_HWNylZ/runs
    - open files    : 1


julia> TensorBoardLogger.write_hparams!(
           lg,
           Dict("hi" => 1.0),
           ["x/val"]
       )

julia> with_logger(lg) do
           @info "x" val=3.0
       end

julia> convert(MVHistory, lg)
ERROR: Summary value of Nothing while deserializing
Stacktrace:
 [1] error(s::String)
   @ Base ./error.jl:35
 [2] summary_type
   @ ~/.julia/packages/TensorBoardLogger/pDb23/src/Deserialization/deserialization.jl:141 [inlined]
 [3] iterate(iter::TensorBoardLogger.SummaryDeserializingIterator, state::Int64)
   @ TensorBoardLogger ~/.julia/packages/TensorBoardLogger/pDb23/src/Deserialization/deserialization.jl:190
 [4] iterate
   @ ~/.julia/packages/TensorBoardLogger/pDb23/src/Deserialization/deserialization.jl:184 [inlined]
 [5] map_summaries(fun::TensorBoardLogger.var"#131#132"{MVHistory{History}}, logdir::TBLogger{String, IOStream}; purge::Bool, tags::Nothing, steps::Nothing, smart::Bool)
   @ TensorBoardLogger ~/.julia/packages/TensorBoardLogger/pDb23/src/Deserialization/deserialization.jl:253
 [6] map_summaries
   @ ~/.julia/packages/TensorBoardLogger/pDb23/src/Deserialization/deserialization.jl:237 [inlined]
 [7] #convert#130
   @ ~/.julia/packages/TensorBoardLogger/pDb23/src/Optional/ValueHistories.jl:5 [inlined]
 [8] convert(::Type{MVHistory}, tbl::TBLogger{String, IOStream})
   @ TensorBoardLogger ~/.julia/packages/TensorBoardLogger/pDb23/src/Optional/ValueHistories.jl:3 [9] top-level scope
   @ REPL[34]:1

In contrast, the following works:

julia> lg = TBLogger(joinpath(mktempdir(), "runs"), min_level=Logging.Info; step_increment=0)
TBLogger:
    - Log level     : Info
    - Current step  : 0
    - Output        : /tmp/jl_u2Pmjk/runs
    - open files    : 1


julia> with_logger(lg) do
           @info "x" val=3.0
       end

julia> convert(MVHistory, lg)
MVHistory{History}
  :x/val => 1 elements {Int64,Float32}

julia> versioninfo()
Julia Version 1.9.3
Commit bed2cd540a1 (2023-08-24 14:43 UTC)
Build Info:
  Official https://julialang.org/ release
Platform Info:
  OS: Linux (x86_64-linux-gnu)
  CPU: 12 × Intel(R) Core(TM) i7-10710U CPU @ 1.10GHz
  WORD_SIZE: 64
  LIBM: libopenlibm
  LLVM: libLLVM-14.0.6 (ORCJIT, skylake)
  Threads: 1 on 12 virtual cores

(jl_ymjBPx) pkg> st --manifest
Status `/tmp/jl_ymjBPx/Manifest.toml`
  [621f4979] AbstractFFTs v1.5.0
  [79e6a3ab] Adapt v3.6.2
  [e1450e63] BufferedStreams v1.2.1
  [3da002f7] ColorTypes v0.11.4
⌅ [c3611d14] ColorVectorSpace v0.9.10
  [5ae59095] Colors v0.12.10
  [34da2185] Compat v4.9.0
  [9a962f9c] DataAPI v1.15.0
  [864edb3b] DataStructures v0.18.15
  [ffbed154] DocStringExtensions v0.9.3
  [4e289a0a] EnumX v1.0.4
  [5789e2e9] FileIO v1.16.1
  [53c48c17] FixedPointNumbers v0.8.4
  [a2bd30eb] Graphics v1.1.2
⌅ [a09fc81d] ImageCore v0.9.4
  [92d709cd] IrrationalConstants v0.2.2
  [692b3bcd] JLLWrappers v1.5.0
  [2ab3a3ac] LogExpFunctions v0.3.26
  [dbb5928d] MappedArrays v0.4.2
  [e1d29d7a] Missings v1.1.0
  [e94cdb99] MosaicViews v0.3.4
  [77ba4419] NaNMath v1.0.2
  [6fe1bfb0] OffsetArrays v1.12.10
  [bac558e1] OrderedCollections v1.6.2
  [5432bcbf] PaddedViews v0.5.12
  [aea7be01] PrecompileTools v1.2.0
  [21216c6a] Preferences v1.4.0
  [3349acd9] ProtoBuf v1.0.14
  [3cdcf5f2] RecipesBase v1.3.4
  [189a3867] Reexport v1.2.2
  [ae029012] Requires v1.3.0
  [a2af1166] SortingAlgorithms v1.1.1
  [276daf66] SpecialFunctions v2.3.1
  [cae243ae] StackViews v0.1.1
  [82ae8749] StatsAPI v1.7.0
  [2913bbd2] StatsBase v0.34.0
  [899adc3e] TensorBoardLogger v0.1.22
  [62fd8b95] TensorCore v0.1.1
  [3bb67fe8] TranscodingStreams v0.9.13
  [98cad3c8] ValueHistories v0.5.4
  [efe28fd5] OpenSpecFun_jll v0.5.5+0
  [0dad84c5] ArgTools v1.1.1
  [56f22d72] Artifacts
  [2a0f44e3] Base64
  [8bf52ea8] CRC32c
  [ade2ca70] Dates
  [f43a241f] Downloads v1.6.0
  [7b1f6079] FileWatching
  [b77e0a4c] InteractiveUtils
  [b27032c2] LibCURL v0.6.3
  [76f85450] LibGit2
  [8f399da3] Libdl
  [37e2e46d] LinearAlgebra
  [56ddb016] Logging
  [d6f4376e] Markdown
  [ca575930] NetworkOptions v1.2.0
  [44cfe95a] Pkg v1.9.2
  [de0858da] Printf
  [3fa0cd96] REPL
  [9a3f8284] Random
  [ea8e919c] SHA v0.7.0
  [9e88b42a] Serialization
  [6462fe0b] Sockets
  [2f01184e] SparseArrays
  [10745b16] Statistics v1.9.0
  [fa267f1f] TOML v1.0.3
  [a4e569a6] Tar v1.10.0
  [8dfed614] Test
  [cf7118a7] UUIDs
  [4ec0a83e] Unicode
  [e66e0078] CompilerSupportLibraries_jll v1.0.5+0
  [deac9b47] LibCURL_jll v7.84.0+0
  [29816b5a] LibSSH2_jll v1.10.2+0
  [c8ffd9c3] MbedTLS_jll v2.28.2+0
  [14a3606d] MozillaCACerts_jll v2022.10.11
  [4536629a] OpenBLAS_jll v0.3.21+4
  [05823500] OpenLibm_jll v0.8.1+0
  [bea87d4a] SuiteSparse_jll v5.10.1+6
  [83775a58] Zlib_jll v1.2.13+0
  [8e850b90] libblastrampoline_jll v5.8.0+0
  [8e850ede] nghttp2_jll v1.48.0+0
  [3f19e933] p7zip_jll v17.4.0+0

Calling the function log_custom_scalar results in an error.

Minimal Working Example

using Pkg
Pkg.activate(mktempdir())
Pkg.add(name="TensorBoardLogger", version="0.1.19")
using TensorBoardLogger
logger = TBLogger("tensorboardlogger-issue-120", tb_overwrite)
log_value(logger, "a", 1)
log_value(logger, "b", 2)
log_custom_scalar(logger, Dict("ab" => Dict("ab" => (tb_multiline, ["a", "b"]))))
# ERROR: UndefVarError: `MultilineChartContent` not defined

This issue is fixed by #119:

# (Restart Julia)
using Pkg
Pkg.activate(mktempdir())
Pkg.add(url="https://github.com/de-souza/TensorBoardLogger.jl", rev="fix-MultilineChartContent-not-defined")
using TensorBoardLogger
logger = TBLogger("tensorboardlogger-issue-120", tb_overwrite)
log_value(logger, "a", 1)
log_value(logger, "b", 2)
log_custom_scalar(logger, Dict("ab" => Dict("ab" => (tb_multiline, ["a", "b"]))))
# OK

@willtebbutt was keen to use this with Turing.jl
I have no real idea where it could be or how.
But I think it would be a great demo to have.

The last call of @info seems not to be flushed. Is this tested or I did something wrong?

Hello, I updated TensorboardLogger to the latest 0.1.21 version, and the following minimum example doesn't work anymore:

using Logging
using TensorBoardLogger
using ValueHistories

logger = TBLogger("logs"; min_level=Logging.Info)

with_logger(logger) do
    @info "train" loss = 1.0
    @info "train" loss = 0.0
end

r = TBReader("logs1")
hist = MVHistory()

TensorBoardLogger.map_summaries(r) do tag, iter, val
    push!(hist, Symbol(tag), iter, val)
end

Here is the error:

ERROR: AssertionError: crc_data == crc_data_ck
Stacktrace:
 [1] read_event(f::IOStream)
   @ TensorBoardLogger ~/.julia/packages/TensorBoardLogger/LO5eP/src/Deserialization/deserialization.jl:49
 [2] iterate(it::TensorBoardLogger.TBEventFileIterator, state::Int64)
   @ TensorBoardLogger ~/.julia/packages/TensorBoardLogger/LO5eP/src/Deserialization/deserialization.jl:126
 [3] map_summaries(fun::var"#21#22", logdir::TBReader; purge::Bool, tags::Nothing, steps::Nothing, smart::Bool)
   @ TensorBoardLogger ~/.julia/packages/TensorBoardLogger/LO5eP/src/Deserialization/deserialization.jl:270
 [4] map_summaries(fun::Function, logdir::TBReader)
   @ TensorBoardLogger ~/.julia/packages/TensorBoardLogger/LO5eP/src/Deserialization/deserialization.jl:248

Reading files written with an older version still works.
Am I missing something ?

ps: by the way, it seems that version 0.1.20 cannot be installed from the general registry for some reason (only from the github repository url)

And the actual info inside TEXT is shown as emtpy with tensorboard complaining: UnicodeDecodeError: 'utf-8' codec can't decode byte ...

I am using a machine with a network file system, and I am following a REPL-based dev flow. When I run the code to initialize the TBLogger (for a second time), I get a "resource busy" message from the OS.

It would be nice to support Base.close(::TBLogger) which performs the first part of reset!(::TBLogger) by releasing all the file handles.

:append, :overwrite, ..., seem more convenient to use on the user side than tb_append, tb_overwrite, ...

Is there a convenient way to load the logged values?

For instance, I've logged a bunch of scalars in the usual way

using TensorBoardLogger, Logging
tb_logger = TBLogger("path/to/logfile")

function cb()
    with_logger(tb_logger) do
        @info "loss" Flux.data(loss())
    end
end

and now I want to plot/analyze those values back inside Julia. What's the recommended way to do this?

Hi,
I am working on a Flux's model zoo example using TensorBoardLogger.jl
FluxML/model-zoo#209
I don't use the automatic step increment, just manually set the step:

TensorBoardLogger.set_step!(tblogger, epoch)
with_logger(tblogger) do
    @info "train" loss=train.loss  acc=train.acc   log_step_increment=0
    @info "test"  loss=test.loss   acc=test.acc    log_step_increment=0
end

In this case, it would be nice to set once and for all the default step size, something like:

TensorBoardLogger.set_step_increment!(tblogger, 0)

Would you consider adding this feature?

Since tensorboard supports logdirs being in s3 directly, it would be nice to be able to write to s3 directly too!

It looks like how Plots.jl and PyPlot.jl are handled is very generic. So, why not accept any object that is showable as image/png? What I mean is something like this:

function log_image(lg::TBLogger, name::AbstractString, obj; step=nothing)
    if !showable("image/png", obj)
        throw(ArgumentError("Cannot be logged as an image: $obj"))
        # or maybe `@error`?
    end
    pb = PipeBuffer()
    show(pb, "image/png", obj)
    img = PNG(pb)
    log_keyval(lg, name, img, step)
end

It would be very useful when creating plots in other libraries like VegaLite.jl which supports show(io, "image/png", plot).

Hi, taking the Readme of this wonderful project seriously:

We also support native logging of the types defined by a few third-party packages, such as Plots, PyPlot and Gadfly plots. If there are other libraries that you think we should include in the list, please open an issue.

Would it be possible to also support Makie / CairoMakie?

The Github Pages deployment points to https://philipvinc.github.io/TensorBoardLogger.jl, whereas the documentation (as far as I can tell) is hosted at https://julialogging.github.io/TensorBoardLogger.jl/dev/

Found the first link at https://github.com/JuliaLogging/TensorBoardLogger.jl/deployments/activity_log?environment=github-pages

Basically if someone passes a matrix to log_histogram,
it would be good to call vec on it.

I believe this is the behavour of tensorboard in TensorFlow.

But not present on PRs or locally. First one I see is https://github.com/JuliaLogging/TensorBoardLogger.jl/runs/2764508008 but it doesn’t look like anything that the commit touched. Reported here: #103 (comment)

I'm trying to use TensorBoardLogger in a multi-threaded environment. Every thread gets its own logger, writing to a different file. But I'm still getting occasional exceptions like the one below (ProtoBuf related ???).

Is there anything else that I should keep in mind?

TaskFailedException:
BoundsError: attempt to access 2×8 BitArray{2} at index [1:2, 9]
Stacktrace:
 [1] throw_boundserror(::BitArray{2}, ::Tuple{UnitRange{Int64},Int64}) at ./abstractarray.jl:537
 [2] checkbounds at ./abstractarray.jl:502 [inlined]
 [3] view at ./subarray.jl:155 [inlined]
 [4] maybeview at ./views.jl:133 [inlined]
 [5] dotview at ./broadcast.jl:1160 [inlined]
 [6] _unset_oneofs(::BitArray{2}, ::Array{Int64,1}, ::Int64) at /home/peter/.julia/packages/ProtoBuf/JY3fQ/src/codec.jl:612
 [7] _fillset(::TensorBoardLogger.Summary_Value, ::Symbol, ::Bool, ::Bool) at /home/peter/.julia/packages/ProtoBuf/JY3fQ/src/codec.jl:641
 [8] fillset at /home/peter/.julia/packages/ProtoBuf/JY3fQ/src/codec.jl:633 [inlined]
 [9] setproperty! at /home/peter/.julia/packages/ProtoBuf/JY3fQ/src/utils.jl:5 [inlined]
 [10] _protobuild(::TensorBoardLogger.Summary_Value, ::Base.Iterators.Pairs{Symbol,Any,Tuple{Symbol,Symbol},NamedTuple{(:tag, :simple_value),Tuple{String,Float32}}}) at /home/peter/.julia/packages/ProtoBuf/JY3fQ/src/utils.jl:61
 [11] Summary_Value at /home/peter/.julia/packages/TensorBoardLogger/gv4oF/src/protojl/summary_pb.jl:63 [inlined]
 [12] scalar_summary at /home/peter/.julia/packages/TensorBoardLogger/gv4oF/src/Loggers/LogValue.jl:19 [inlined]
 [13] log_metric(::TensorBoardLogger.TBLogger, ::Symbol, ::Float32, ::Int64, ::DateTime) at /home/peter/dev/Quanta.jl/test/multi.jl:5
 [14] macro expansion at /home/peter/dev/Quanta.jl/test/multi.jl:21 [inlined]
 [15] (::var"#29#threadsfor_fun#8"{UnitRange{Int64}})(::Bool) at ./threadingconstructs.jl:81
 [16] (::var"#29#threadsfor_fun#8"{UnitRange{Int64}})() at ./threadingconstructs.jl:48
wait at task.jl:267 [inlined]
threading_run(::Function) at threadingconstructs.jl:34
macro expansion at threadingconstructs.jl:93 [inlined]
test123() at multi.jl:15
top-level scope at multi.jl:27
include_string(::Function, ::Module, ::String, ::String) at loading.jl:1088

I think now is a good time to register the package,
as v0.1.0
so still Alpha by semver standards

returning the device (e.g. :cpu, or :cuda_0) where a model lives

This is a list of all Tensorboard plugins and types that they (potentially) can accept. I want to see the overlap among different types.

• Scalar (scalar vs iteration line plot)
  • accepts <:Number. Non Real numbers can be preprocessed (e.g. Complex numbers);
• Scalars (taken from TensorboardX, displays multiple curves on the same scalar plot)
  • ? potentially a N-tuple of numbers ?
• Histograms/Distributions
  • Tuple{Vector{Real},Vector{Real}} for histogram's bin edges and bin heights;
  • Vector, plot a vector as an histogram by converting data to Tuple(1:length(data)-1, data) ;
• Images
  • 3-Arrays are naturally 2D images;
  • Matrix is a grayscale 2D image;
  • Vector is a grayscale 1D image;
  • Could also be passed as an image object (jpeg, eps...)
• Text
  • String
  • Vector{String} or Matrix{String} are also rendered as lists or tables.
• Embeddings
  • Requires a Tensor with data, and optionally label_imgs and metadata
• PR Curves
  • ??
• Audio
  • This is specific enough and you must specify the sampling rate and other parameters that I guess can be handled by a specific type.

Extra:
(This is something I need for myself and is not currently supported by TensorBoard). Eventually I would like to contribute to tensorboard a plugin to show a whole plot/curve at each iteration (very similar to what PR curve does). Tensorboard dev team is also considering this but apparently they don't have the time to work on it ATM.

• Curves
  • Tuple{Vector{Real},Vector{Real}} (this is different from histograms, as the length of the two vectors is the same in this case while for histograms they differ by 1).

It would mean that logging graphs would only be enabled in you had LightGraphs loaded.

But then we can cut down our dependencies significantly.

I just realised that if you log

julia> mri = testimage("mri")
julia> eltype(mri)
ColorTypes.Gray{FixedPointNumbers.Normed{UInt8,8}}
julia> size(mri)
(226, 186, 27)
julia> log_image(logger, "image/explicitinterface/mri", mri, step = 0) #either
julia> @info "image/loginterface" mri

Tensorboard will display a single image under the tag image/explicitinterface/mri, and a slider with 27 values that let you scan across all observations.

Instead, if we log

julia> noise = rand(226, 186, 3, 27) #Format is HWCN
Julia> log_image(logger, "image/explicitinterface/noise", noise, HWCN, step = 0)
julia> @info "image/loginterface" noise=TBImage(noise, HWCN)

Tensorboard will display 27 different images under 27 different tags image/loginterface/noise/1....27.

The reason is that under the hood, we have special logic for AbstractArray{<:Colorant} with 3 dimensions with the same tag so that you can slide through them, but we don't do this for 4-dimensional abstract arrays.

As we also have the wrapper TBImages which logs an array of N images with the same format under the same tag by appending a 1...N identifier at the end of the tag, I think that interpreting the observation (N in HWCN) channel differently among AbstractArray{<:Colorant} and abstractArray{<:Number} is an inconsistency.

I don't have the experience to know if this is indeed inconsistent, or if people that work with image data more than me find this useful. In case everyone believes this is an inconsistency, I could quickly fix this in #42.

cc @oxinabox, @shashikdm @xukai92

the serialization bug fix #132 has yet to be released.

This issue is used to trigger TagBot; feel free to unsubscribe.

If you haven't already, you should update your TagBot.yml to include issue comment triggers.
Please see this post on Discourse for instructions and more details.

Since TBL already supports (bins,weights), it would be cool it it could support StatsBase.Histogram directly as well.

When we implement the AbstractLogger interface, we need to find a way to pass the step (iteration) count to tensorboard. This can be done nicely by passing it as a key-value pair in the logged message, but I am interested in the case where this is not done.

The obvious solution is to store a global iteration count in the logger itself, but how do we update it? Defining a set_step would make the code break if one changes Logger. Maybe if one logs step this will not be logged and just used to update the internal counter?

The behavour of TensorFlow.jl if you have a logger set to path mypath
is to try mypath2 then mypath3 and so forth until it finds a free name.

It might make sense to have overwrite take 3 options:

• Append (current behavour if false)
• Overwrite (current behavour if true)
• IncrementFilename (my suggested behavour)

Right now the method that one needs to implement to extend TBLogger for his own custom types is preprocess(tag, value, data) and it modifies the data argument.

I find this confusing. It would be more Julian if it was defined as

preprocess!(data, tag, value)

And maybe even the preprocess! could be changed to something that is more clear.
As adding the ! would change the function name, this technically would just be a deprecation and not a breaking change, so it should not be a big issue for a v0.2.

Like Complex breaks down into two scalar plots,

For

struct Foo
   bar1::Bar
   bar2::Bar
end
struct Bar 
    x::Vector
    y::Real
    z::Real
end

Then a Foo in the above should break down into 4 scalar plots and 2 histograms.
With everything named approprately. e,g, a Foo/bar1/y, Foo/bar2/x etc.

This would allow easy logging of all the weights and baises in a Flux model.
probably also need a few other things like the ability to have a preprocess function (Tracker.data for Flux`),
and #4

I don't know if this is expected to work or not, but I serialized a TBLogger with the BSON package and then deserialized it in another Julia session and ended up getting errors because the file handles were bad. I created a work around, but is this something that TBLogger could handle internally? For example, it could check file handles are valid and re-open the files if needed.

Maybe by cleaning up the following:

flat(xs) = reduce(vcat, vec.(data.(xs)))

function train!(loss_func, mdl, features, outputs, settings)
    training_record = NamedTuple{(:loss,),Tuple{Float32}}[]

    network_parameters = params(mdl)
    @assert length(network_parameters) > 0

    if LOG_TENSORBOARD
        lg = TBLogger("logs/run"; min_level=Logging.Debug)
    else
        lg = current_logger()
    end
    with_logger(lg) do
        for iter in 1:settings.max_epochs
            grad = gradient(network_parameters) do
                preds = mdl(features)
                training_loss = loss_func(preds, outputs, settings)
                push!(training_record, (loss=data(training_loss),))

                @debug "" network_parameters = flat(network_parameters)
                @debug "" generated_volumes = vec(data(volumes))
                @debug "" training_loss = data(training_loss)

                iter += 1
                return training_loss
            end
            @debug "" mean_grad=mean(abs, flat(values(grad.grads)))
            update!(settings.optimizer, network_parameters, grad)
            last_loss = training_record[end].loss
        end
    end
    return training_record
end

Logger fails silently when logging from multithreaded for loop / map, can't communicate with logger afterward.
Julia default logger works well in a similar situation.

MWE:

using TensorBoardLogger
using Logging
tblogger = TBLogger("MWE_tblogger_threads", min_level=Logging.Info)

with_logger(tblogger) do
  for i in 1:10
    @info "blah" i^2
  end
  @info "finished sequential" finished=true
end # works well

with_logger(tblogger) do
  Threads.@threads for i in 1:10
    println(i^2) # no logging here
  end
  @info "finished threads" finished=true
end # works well too

with_logger(tblogger) do
  Threads.@threads for j in 1:10
    @info "blah" j^2
  end
  @info "finished threads" finished=false
end # silently fails, and doesn't log anything

with_logger(tblogger) do
  @info "can't communicate" something=1
end # can't communicate with logger afterward

After updating to ProtoBuf 1.0.0 #124 I found that summaries are not logged correctly to Tensorboard.
Some of them do get logged but some don't. I suspect that's because some summaries are fine but after trying to log incorrectly with some of them, the file or tensorboard stops registring the ones following that.

I prepared a minimal reproducing code by revising the Flux example to the new Flux API (the existing example uses a deprecated API)
nomadbl@fc9ba3e

During logging I observe an error message

[2023-07-01T23:12:43Z WARN  rustboard_core::run] Read error in ./content/log/events.out.tfevents.1.68825314069885e9.lior-HP-Pavilion-Laptop-15-cs3xxx: ReadRecordError(BadLengthCrc(ChecksumError { got: MaskedCrc(0x85987b32), want: MaskedCrc(0x00000000) }))

Which after some googling I can only speculate it indicates it has something to do with multiprocessing and the file trying to get written by multiple instances of the logger in different threads.
So far I tried (without success) to fix it under that assumption by specifying the logger should lock the file:
src/TBLogger.jl, 119:
file = open(fpath, "w"; lock=true)

Any other ideas or insights are welcome. I'll try to isolate the issue using the above mentioned reproducing code.

Following #8 and #9, would it be feasible to integrate an API along the lines of:

...
with_logger(lg) do
    for i=1:100
        # increments log_step_increment by one by default
        # could also take a custom increment as an optional parameter
        with_step() do
            @info "test" i=i        
        end
    end
end

For use-cases where one only wants to increment the step once per iteration (e.g. to match tensorflow's behaviour), this would eliminate the need to specify log_step_increment=0 on the remaining N-1 logging calls.

We should think about more on how we display things we don't have special overloads for.
the generric fallbacks.

I present two options:

ShowMime Option

#49 Which suggested the generic fallback for anything that is showable("img/png", ...)
should be to show it as an image in the image view.

And further my own desire that anything with showable for mimetype "text/plain" , "text/markdown", or "text/html" be displayed in the text view (which can render markdown/html); and that the ultimate fallback be to just showing (repring) like the ConsoleLogger does.

Destructure Option (current behavour)

This is mutually exclusive with our current fallback behavour of destructuring all structs.
E.g. to convert struct Foo a b end into showing a and showing b.
Which is also a good default.

One possibility to resolve this is to have wrapper types for one of the behavours.
So when logging something you want to have the particular behavour for, that does not agree with the default you wrap it in some indicator type.

I like the idea of making destructure a wrapper, and defaulting to the ShowMime option.
A particular advantage of this is one could use it to force destructurring on something we might have a default for. I guess something similar applies to show.
And later we might want to exend that, to be parameterized by how deep to destructure.

Are there any plans to add some Julia-based visualization of the log files (as an alternative to tensorboard)?

I tried the following simple example for the log file of the usage example in the readme.
I think with @essenciary's Stipple.jl one could get something nice here.

using TensorBoardLogger, Genie, Genie.Router, Genie.Renderer.Html,
Stipple, StippleCharts, StippleCharts.Charts

function pushdata!(model, i, v)
    model.i = i
    push!(model.plot_data.val[1].plotdata.data, v)
end

function loaddata(path, var, model; timeout = 1)
    while true
        map_summaries((s, i, v) -> s == var ? pushdata!(model, i, v) : nothing,
                      path, steps = model.i + 1 : typemax(UInt))
        sleep(timeout)
    end
end

Base.@kwdef mutable struct Dashboard <: ReactiveModel
    i::Int = 0
    plot_option::PlotOptions = PlotOptions(chart_type = :scatter)
    plot_data::R{Vector{PlotSeries}} = [PlotSeries(name = "test/j",
                                                   plotdata = PlotData(Float32[]))]
end

Stipple.register_components(Dashboard, StippleCharts.COMPONENTS)
model = Stipple.init(Dashboard)

function ui()
  page(
    root(model), class="container", [
        row([
        cell(class="st-module", [
          h5("Series")
          plot(:plot_data; options = :plot_option)
        ])
      ])
    ], title="TBoardLogger"
  ) |> html
end

route("/", ui)

up()

loaddata("logs/run", "test/j", model) # no auto-refresh of the browser...

Custom Scalars plugin produces an error.

Minimal working example:

julia> using TensorBoardLogger

julia> lg = TBLogger("logs", tb_overwrite)
TBLogger:
    - Log level     : Info
    - Current step  : 0
    - Output        : D:\Downloads\Video\logs
    - open files    : 1


julia> layout = Dict("Cat" => Dict("Curve" => (tb_multiline, ["Curve/1", "Curve/2"])))
Dict{String, Dict{String, Tuple{TensorBoardLogger.tb_chart_type, Vector{String}}}} with 1 entry:
  "Cat" => Dict("Curve"=>(tb_multiline, ["Curve/1", "Curve/2"]))

julia> log_custom_scalar(lg, layout)
ERROR: MethodError: no method matching TensorBoardLogger.tensorboard_plugin_custom_scalar.MultilineChartContent(; tag::Vector{String})

Closest candidates are:
  TensorBoardLogger.tensorboard_plugin_custom_scalar.MultilineChartContent(::Vector{String}) got unsupported keyword argument "tag"
   @ TensorBoardLogger C:\users\user\.julia\packages\TensorBoardLogger\pDb23\src\protojl\tensorboard\plugins\custom_scalar\tensorboard\layout_pb.jl:12
  TensorBoardLogger.tensorboard_plugin_custom_scalar.MultilineChartContent(::Any) got unsupported keyword argument "tag"
   @ TensorBoardLogger C:\users\user\.julia\packages\TensorBoardLogger\pDb23\src\protojl\tensorboard\plugins\custom_scalar\tensorboard\layout_pb.jl:12

Stacktrace:
  [1] chart(name::String, metadata::Tuple{TensorBoardLogger.tb_chart_type, Vector{String}})
    @ TensorBoardLogger C:\users\user\.julia\packages\TensorBoardLogger\pDb23\src\Loggers\LogCustomScalar.jl:28
  [2] #72
    @ .\none:0 [inlined]
  [3] iterate
    @ .\generator.jl:47 [inlined]
  [4] collect(itr::Base.Generator{Base.Iterators.Zip{Tuple{Base.KeySet{String, Dict{String, Tuple{TensorBoardLogger.tb_chart_type, Vector{String}}}}, Base.ValueIterator{Dict{String, Tuple{TensorBoardLogger.tb_chart_type, Vector{String}}}}}}, TensorBoardLogger.var"#72#73"})
    @ Base .\array.jl:782
  [5] charts
    @ C:\users\user\.julia\packages\TensorBoardLogger\pDb23\src\Loggers\LogCustomScalar.jl:42 [inlined]
  [6] (::TensorBoardLogger.var"#74#75")(::Tuple{String, Dict{String, Tuple{TensorBoardLogger.tb_chart_type, Vector{String}}}})
    @ TensorBoardLogger .\none:0
  [7] iterate
    @ .\generator.jl:47 [inlined]
  [8] collect(itr::Base.Generator{Base.Iterators.Zip{Tuple{Base.KeySet{String, Dict{String, Dict{String, Tuple{TensorBoardLogger.tb_chart_type, Vector{String}}}}}, Base.ValueIterator{Dict{String, Dict{String, Tuple{TensorBoardLogger.tb_chart_type, Vector{String}}}}}}}, TensorBoardLogger.var"#74#75"})
    @ Base .\array.jl:782
  [9] custom_scalar_summary(layout::Dict{String, Dict{String, Tuple{TensorBoardLogger.tb_chart_type, Vector{String}}}})
    @ TensorBoardLogger C:\users\user\.julia\packages\TensorBoardLogger\pDb23\src\Loggers\LogCustomScalar.jl:47
 [10] log_custom_scalar(logger::TBLogger{String, IOStream}, layout::Dict{String, Dict{String, Tuple{TensorBoardLogger.tb_chart_type, Vector{String}}}}; step::Nothing)
    @ TensorBoardLogger C:\users\user\.julia\packages\TensorBoardLogger\pDb23\src\Loggers\LogCustomScalar.jl:20
 [11] log_custom_scalar(logger::TBLogger{String, IOStream}, layout::Dict{String, Dict{String, Tuple{TensorBoardLogger.tb_chart_type, Vector{String}}}})
    @ TensorBoardLogger C:\users\user\.julia\packages\TensorBoardLogger\pDb23\src\Loggers\LogCustomScalar.jl:19
 [12] top-level scope
    @ REPL[8]:1

julia>

Version Information:

julia> versioninfo()
Julia Version 1.9.3
Commit bed2cd540a (2023-08-24 14:43 UTC)
Build Info:
  Official https://julialang.org/ release
Platform Info:
  OS: Windows (x86_64-w64-mingw32)
  CPU: 16 × AMD Ryzen 7 PRO 5850U with Radeon Graphics
  WORD_SIZE: 64
  LIBM: libopenlibm
  LLVM: libLLVM-14.0.6 (ORCJIT, znver3)
  Threads: 1 on 16 virtual cores

Package Status:

(temp) pkg> st
Status `D:\Downloads\Video\temp\Project.toml`
  [899adc3e] TensorBoardLogger v0.1.22

The docs are currently broken.

Hi,
while the example under Basic Usage works fine, this code produces empty event files:

using TensorBoardLogger, Logging, Random

lg=TBLogger("tensorboard_logs/run", min_level=Logging.Info)

with_logger(lg) do
    for i=1:100
        @info "scalar" i=i j=i^2 
    end 
end

File content:

$ tensorboard --inspect --logdir tensorboard_logs/run
TensorFlow installation not found - running with reduced feature set.
======================================================================
Processing event files... (this can take a few minutes)
======================================================================

Found event files in:
tensorboard_logs/run

These tags are in tensorboard_logs/run:
audio -
histograms -
images -
scalars -
tensor -
======================================================================

Event statistics for tensorboard_logs/run:
audio -
graph -
histograms -
images -
scalars -
sessionlog:checkpoint -
sessionlog:start -
sessionlog:stop -
tensor -

As the title explains, I'm running

using TensorBoardLogger
using Logging

tb_logger = TBLogger("tensorboard_logs/run", min_level=Logging.Debug)
with_logger(tb_logger) do
    @info "loss" loss=10.
end

and get

ERROR: LoadError: StackOverflowError:
Stacktrace:
 [1] fieldnames at ./reflection.jl:173 [inlined]
 [2] propertynames at ./reflection.jl:1371 [inlined]
 [3] logable_propertynames at /root/.julia/packages/TensorBoardLogger/tMpIk/src/logger_dispatch.jl:39 [inlined]
 [4] preprocess(::String, ::Type{TensorBoardLogger.tensorboard.AllocationDescription}, ::Array{Pair{String,Any},1}) at /root/.julia/packages/TensorBoardLogger/tMpIk/src/logger_dispatch.jl:16
 [5] preprocess(::String, ::Module, ::Array{Pair{String,Any},1}) at /root/.julia/packages/TensorBoardLogger/tMpIk/src/logger_dispatch.jl:19 (repeats 20113 times)
 [6] handle_message(::TBLogger, ::Base.CoreLogging.LogLevel, ::String, ::Module, ::Symbol, ::Symbol, ::String, ::Int64; kwargs::Base.Iterators.Pairs{Symbol,ProtoBuf.ProtoMeta,Tuple{Symbol},NamedTuple{(:meta,),Tuple{ProtoBuf.ProtoMeta}}}) at /root/.julia/packages/TensorBoardLogger/tMpIk/src/TBLogger.jl:241
 [7] macro expansion at ./logging.jl:332 [inlined]
 [8] writeproto(::Base.GenericIOBuffer{Array{UInt8,1}}, ::TensorBoardLogger.tensorboard.Event, ::ProtoBuf.ProtoMeta) at /root/.julia/packages/ProtoBuf/NJbMU/src/codec.jl:420
 [9] writeproto at /root/.julia/packages/ProtoBuf/NJbMU/src/codec.jl:419 [inlined]
 [10] write_event(::IOStream, ::TensorBoardLogger.tensorboard.Event) at /root/.julia/packages/TensorBoardLogger/tMpIk/src/event.jl:31
 [11] handle_message(::TBLogger, ::Base.CoreLogging.LogLevel, ::String, ::Module, ::Symbol, ::Symbol, ::String, ::Int64; kwargs::Base.Iterators.Pairs{Symbol,Float64,Tuple{Symbol},NamedTuple{(:loss,),Tuple{Float64}}}) at /root/.julia/packages/TensorBoardLogger/tMpIk/src/TBLogger.jl:250
 [12] macro expansion at ./logging.jl:332 [inlined]
 [13] (::var"#1#2")() at /home/jupyter/project/scripts/train.jl:6
 [14] with_logstate(::Function, ::Any) at ./logging.jl:408
 [15] with_logger(::Function, ::TBLogger) at ./logging.jl:514
 [16] top-level scope at /home/jupyter/project/scripts/train.jl:11
 [17] include(::Function, ::Module, ::String) at ./Base.jl:380
 [18] include(::Module, ::String) at ./Base.jl:368
 [19] exec_options(::Base.JLOptions) at ./client.jl:296
 [20] _start() at ./client.jl:506
in expression starting at /home/jupyter/project/scripts/train.jl:5

And my environment:

(@v1.5) pkg> status
Status `~/.julia/environments/v1.5/Project.toml`
  [7073ff75] IJulia v1.23.2
  [899adc3e] TensorBoardLogger v0.1.15
  [cf7118a7] UUIDs

https://github.com/PhilipVinc/TensorBoardLogger.jl/blob/08f57d854ca1b32cc4cfb0a24881d57ec9f5bb1c/src/Loggers/LogValue.jl#L8

Hello, in contrary to the description and probably the expected behavior, the method log_value sets the argument step to nothing on default. As suggested in the description, a more natural default value for the argument would be step(logger). I believe this was intended but somehow accidentally omitted during development.

Cheers,

It is very useful to have hyperparameters when looking at logs. i.e. https://pytorch.org/docs/stable/tensorboard.html#torch.utils.tensorboard.writer.SummaryWriter.add_hparams

Is this possible to add to the TensorBoardLogger.jl? Either giving a dictionary or just calling yourself on each dictionary item.