Fix crash when deleting an actor

The Cleanup method is called on actors when they exit

actor.go

@@ -3,20 +3,40 @@ package camfish
 import "context"
 
 // Actor is a participant in the environment. All public methods on an actor
-// must be safe for concurrent use by multiple goroutines. Additionally, any
+// should be safe for concurrent use by multiple goroutines except for AddFlags,
-// type which explicitly implements Actor should:
+// Init, Configure, and ProcessConfig. Additionally, any type which explicitly
+// implements Actor should:
+//
 //   - Treat all public fields, values, indices, etc. as immutable
 //   - Satisfy Actor as a pointer, not a value
 //   - Not have a constructor
+//
+// The CAMFISH environment will use interfaces in this package to probe actors
+// for methods. If an actor is supposed to fulfill one of these interfaces, this
+// should be enforced at compile-time by assigning the actor to an anonymous
+// global variable of that interface type. For instance, this line will ensure
+// that SomeActor fulfills [Resettable]:
+//
+//   var _ camfish.Resettable = new(SomeActor)
 type Actor interface {
-	// Type returns the type name of the actor. The value returned from this
+	// Type returns the "type name" of the actor. The value returned from
-	// is used to locate actors capable of performing a specific task, so it
+	// this is used to locate actors capable of performing a specific task,
-	// absolutely must return the same string every time. Actors implemented
+	// so it absolutely must return the same string every time. It is
-	// in packages besides this one (i.e. not camfish) must not return the
+	// usually best to have this be unique to each actor. Actors implemented
-	// string "cron".
+	// in packages other than this one
+	// (git.tebibyte.media/sashakoshka/camfish) must not return the string
+	// "cron".
 	Type() string
 }
 
+// Named is any object with a name.
+type Named interface {
+	// Name returns the name. This doesn't need to be the same as Type. It
+	// must return the same string every time. It is used to differentiate
+	// actors of the same type in logs.
+	Name() string
+}
+
 // FlagAdder is any object that can add [Flag]s to a [FlagSet]. Actors which
 // implement this interface will be called upon to add flags during and only
 // during the flag parsing phase.
@@ -88,3 +108,29 @@ type Resettable interface {
 	// invalid and any process which depends on it should be shut down.
 	Reset(ctx context.Context) error
 }
+
+// RunShutdownable is any object that needs a context in order to shut down.
+// Actors which implement this interface cannot implement the Runnable
+// interface. This can be used to run an http.Server as an actor.
+type RunShutdownable interface {
+	// Run is similar to [Runnable.Run], but takes no context and blocks
+	// until Shutdown has run and exited. It may also return when something
+	// goes wrong and it cannot continue, in which case it must return a
+	// non-nil error explaining why. Shutdown does not need to be called in
+	// the latter case.
+	Run() error
+	// Shutdown shuts down the actor. It must unblock Run in all cases even
+	// on failure, context expiration, etc. Shutdown must return when or
+	// before the context expires, and must return ctx.Err if there is no
+	// other error to be returned. If Shutdown returns any error, the object
+	// must be treated as invalid and any other process which depends on it
+	// should be shut down.
+	Shutdown(ctx context.Context) error
+}
+
+// Cleanupable is any object that must be cleaned up after it has stopped for
+// good. Actors which implement this interface will be cleaned up after they
+// are deleted from the environment.
+type Cleanupable interface {
+	Cleanup(ctx context.Context) error
+}

actorset.go

@@ -16,6 +16,7 @@ type actorSets struct {
 	configurable    actorSet[Configurable]
 	initializable   actorSet[Initializable]
 	runnable        actorSet[Runnable]
+	runShutdownable actorSet[RunShutdownable]
 	trimmable       actorSet[Trimmable]
 }
 
@@ -52,6 +53,7 @@ func (sets *actorSets) All() iter.Seq[actorSetIface] {
 		yield(&sets.configurable)
 		yield(&sets.initializable)
 		yield(&sets.runnable)
+		yield(&sets.runShutdownable)
 		yield(&sets.trimmable)
 	}
 }
@@ -66,7 +68,9 @@ func (this *actorSets) add(ctx context.Context, actor Actor) {
 		done:  done,
 		order: this.nextOrder,
 	}
-	if _, ok := actor.(Runnable); ok {
+	_, isRunnable        := actor.(Runnable)
+	_, isRunShutdownable := actor.(RunShutdownable)
+	if isRunnable || isRunShutdownable {
 		info.stopped = make(chan struct { })
 	}
 	this.inf[actor] = info

cron.go

@@ -7,6 +7,7 @@ import "context"
 // tasks on time intervals.
 type cron struct {
 	trimFunc func() bool
+	fastTiming bool
 	timing struct {
 		trimInterval time.Duration
 	}
@@ -28,6 +29,9 @@ func (this *cron) Configure (config Config) error {
 		}
 		this.timing.trimInterval = value
 	}
+	if this.fastTiming {
+		this.timing.trimInterval = time.Second * 10
+	}
 	return nil
 }
 

environment.go

@@ -19,6 +19,7 @@ const defaultRestartInitialInterval  =  8 * time.Second
 const defaultRestartInitialIncrement =  8 * time.Second
 const defaultRestartInitialMaximum   =  1 * time.Hour
 const defaultResetTimeout            =  8 * time.Minute
+const defaultCleanupTimeout          =  1 * time.Minute
 const defaultTrimInterval            =  1 * time.Minute
 const defaultTrimTimeout             =  1 * time.Minute
 const defaultShutdownTimeout         =  8 * time.Minute
@@ -33,6 +34,7 @@ type environment struct {
 	done        context.CancelCauseFunc
 	group       sync.WaitGroup
 	conf        MutableConfig
+	cron        *cron
 
 	// flags stores information from built-in flags.
 	flags struct {
@@ -41,6 +43,9 @@ type environment struct {
 		logDirectory string
 		configFile   string
 		verbose      bool
+		crash        bool
+		crashOnError bool
+		fastTiming   bool
 	}
 
 	// running stores whether the environment is currently running.
@@ -58,6 +63,7 @@ type environment struct {
 		restartIntervalIncrement atomicDuration
 		restartIntervalMaximum   atomicDuration
 		resetTimeout             atomicDuration
+		cleanupTimeout           atomicDuration
 		trimTimeout              atomicDuration
 		shutdownTimeout          atomicDuration
 	}
@@ -78,10 +84,11 @@ func (this *environment) Run(name, description string, actors ...Actor) {
 	this.name = name
 	this.description = description
 	this.actors = usync.NewRWMonitor(&actorSets { })
-	this.addToSets(actors...)
+	this.cron = &cron {
-	this.addToSets(&cron {
 		trimFunc: this.phase70_5Trimming,
-	})
+	}
+	this.addToSets(actors...)
+	this.addToSets(this.cron)
 	
 	if !this.phase10FlagParsing()              { os.Exit(2) }
 	if !this.phase13PidFileCreation()          { os.Exit(1) }
@@ -110,7 +117,12 @@ func (this *environment) Add(ctx context.Context, actors ...Actor) error {
 		}
 	}
 	err := this.initializeActors(ctx, initializable...)
-	if err != nil { return err }
+	if err != nil {
+		if this.flags.crashOnError {
+			panic(fmt.Sprint(err))
+		}
+		return err
+	}
 	for _, actor := range actors {
 		if actor, ok := actor.(Configurable); ok {
 			err := actor.Configure(this.conf)
@@ -122,7 +134,9 @@ func (this *environment) Add(ctx context.Context, actors ...Actor) error {
 		}
 	}
 	for _, actor := range actors {
-		if actor, ok := actor.(Runnable); ok {
+		_, isRunnable        := actor.(Runnable)
+		_, isRunShutdownable := actor.(RunShutdownable)
+		if isRunnable || isRunShutdownable {
 			this.start(actor)
 		}
 	}
@@ -134,11 +148,15 @@ func (this *environment) Del(ctx context.Context, actors ...Actor) error {
 	channels := []<- chan struct { } { }
 	for _, actor := range actors {
 		info := this.info(actor)
-		if info.done != nil {
+		if info.stopped != nil {
 			channels = append(channels, info.stopped)
 		}
+		if info.done != nil {
+			info.done()
+		}
 	}
 	for _, channel := range channels {
+		if channel == nil { continue }
 		select {
 		case <- channel:
 		case <- ctx.Done():
@@ -213,7 +231,7 @@ func (this *environment) info(actor Actor) actorInfo {
 // start increments the wait group by one and starts the given actor in the
 // background, restarting it if it fails. this function will exit immediately.
 // see the documentation for run for details.
-func (this *environment) start(actor Runnable) {
+func (this *environment) start(actor Actor) {
 	this.group.Add(1)
 	go this.run(actor)
 }
@@ -223,14 +241,31 @@ func (this *environment) start(actor Runnable) {
 // environment once this function exits, and the environment's wait group
 // counter will be decremented. note that this function will never increment the
 // wait group counter, so start should usually be used instead.
-func (this *environment) run(actor Runnable) {
+func (this *environment) run(actor Actor) {
+	typ := actor.Type()
+	
 	// clean up when done
-	defer this.group.Done()
+	defer func() {
+		this.group.Done()
+		this.delFromSets(actor)
+		if actor, ok := actor.(Cleanupable); ok {
+			ctx, done := context.WithTimeout(
+				context.Background(),
+				defaul(
+					this.timing.cleanupTimeout.Load(),
+					defaultCleanupTimeout))
+			defer done()
+			err := actor.Cleanup(ctx)
+			if err != nil {
+				log.Printf("XXX [%s] failed to cleanup: %v", typ, err)
+				if this.flags.crashOnError {
+					panic(fmt.Sprint(err))
+				}
+			}
+		}
+	}()
 
 	// logging
-	acto, ok := actor.(Actor)
-	if !ok { return }
-	typ := acto.Type()
 	if this.Verb() { log.Printf("(i) [%s] running", typ) }
 	var stopErr error
 	var exited  bool
@@ -243,14 +278,37 @@ func (this *environment) run(actor Runnable) {
 			}
 		} else {
 			log.Printf("!!! [%s] stopped with error: %v", typ, stopErr)
+			if this.flags.crashOnError {
+				panic(fmt.Sprint(stopErr))
+			}
 		}
 	}()
 
 	// contains context information
-	info := this.info(acto)
+	info := this.info(actor)
 	ctx := info.ctx
-	defer close(info.stopped)
+	defer func() {
+		if info.stopped != nil {
+			close(info.stopped)
+		}
+	}()
+	
+	switch actor := actor.(type) {
+	case Runnable:
+		stopErr, exited = this.runRunnable(ctx, actor)
+	case RunShutdownable:
+		stopErr, exited = this.runRunnable(ctx, &runShutdownableShim {
+			shutdownTimeout: defaul(this.timing.shutdownTimeout.Load(), defaultShutdownTimeout),
+			underlying: actor,
+		})
+	default:
+		panic("actor was neither Runnable or RunShutdownable")
+	}
+}
 
+// runRunnable runs an actor implementing [Runnable]. this should only be called
+// from within [environment.run].
+func (this *environment) runRunnable(ctx context.Context, actor Runnable) (stopErr error, exited bool) {
 	// timing
 	restartThreshold         := defaul(this.timing.restartThreshold.Load(), defaultRestartThreshold)
 	restartInitialInterval   := defaul(this.timing.restartInitialInterval.Load(), defaultRestartInitialInterval)
@@ -259,11 +317,19 @@ func (this *environment) run(actor Runnable) {
 	resetTimeout             := defaul(this.timing.resetTimeout.Load(), defaultResetTimeout)
 	restartInterval          := restartInitialInterval
 
-	// main loop
+	acto, ok := actor.(Actor)
+	if !ok { return }
+	typ := acto.Type()
+	
 	for {
 		// run actor
 		lastStart := time.Now()
-		err := panicWrap(ctx, actor.Run)
+		var err error
+		if this.flags.crash {
+			err = actor.Run(ctx)
+		} else {
+			err = panicWrapCtx(ctx, actor.Run)
+		}
 
 		// detect context cancellation
 		if ctxErr := ctx.Err(); ctxErr != nil {
@@ -279,12 +345,18 @@ func (this *environment) run(actor Runnable) {
 			return
 		} else {
 			// failure
-			log.Printf("XXX [%s] failed", typ)
+			log.Printf("XXX [%s] failed: %v", typ, err)
+			if this.flags.crashOnError {
+				panic(fmt.Sprint(err))
+			}
 		}
 
 		// restart logic
 		if time.Since(lastStart) < restartThreshold {
 			log.Printf("!!! [%s] failed too soon, restarting in %v", typ, restartInterval)
+			if this.flags.crashOnError {
+				panic("failed too soon")
+			}
 			timer := time.NewTimer(restartInterval)
 			select {
 			case <- timer.C:
@@ -310,6 +382,9 @@ func (this *environment) run(actor Runnable) {
 				err := actor.Reset(ctx)
 				if err != nil {
 					log.Printf("XXX [%s] failed to reset", typ)
+					if this.flags.crashOnError {
+						panic("failed to reset")
+					}
 				}
 			}()
 			if this.Verb() { log.Printf(".// [%s] reset", typ) }
@@ -369,6 +444,7 @@ func (this *environment) applyConfig() error {
 		}
 		return nil
 	}
+	// TODO: trim interval
 	err := parseDuration("init-timeout", &this.timing.initTimeout)
 	if err != nil { return err }
 	err = parseDuration("restart-threshold", &this.timing.restartThreshold)
@@ -381,9 +457,37 @@ func (this *environment) applyConfig() error {
 	if err != nil { return err }
 	err = parseDuration("reset-timeout", &this.timing.resetTimeout)
 	if err != nil { return err }
+	err = parseDuration("cleanup-timeout", &this.timing.cleanupTimeout)
+	if err != nil { return err }
 	err = parseDuration("trim-timeout", &this.timing.trimTimeout)
 	if err != nil { return err }
 	err = parseDuration("shutdown-timeout", &this.timing.shutdownTimeout)
 	if err != nil { return err }
+
+	if this.flags.fastTiming {
+		this.timing.shutdownTimeout.Store(time.Second * 10)
+	}
+	
 	return nil
 }
+
+type runShutdownableShim struct {
+	underlying RunShutdownable
+	shutdownTimeout time.Duration
+}
+
+func (this *runShutdownableShim) Type() string {
+	return this.underlying.(Actor).Type()
+}
+
+func (this *runShutdownableShim) Run(ctx context.Context) error {
+	ctx, done := context.WithCancel(ctx)
+	defer done()
+	go func() {
+		<- ctx.Done()
+		shutdownCtx, done := context.WithTimeout(context.Background(), this.shutdownTimeout)
+		defer done()
+		this.underlying.Shutdown(shutdownCtx)
+	}()
+	return this.underlying.Run()
+}

examples/broken/main.go

@@ -0,0 +1,27 @@
+// Example broken demonstrates how the environment will forcibly kill the
+// program if an actor cannot shut down.
+package main
+
+import "os"
+import "log"
+import "context"
+import "git.tebibyte.media/sashakoshka/camfish"
+
+func main() {
+	camfish.Run("broken",
+		"Example broken demonstrates how the environment will " +
+		"forcibly kill the program if an actor cannot shut down",
+		new(broken))
+}
+
+// broken is an incorrectly implemented actor that cannot shut down.
+type broken struct { }
+var _ camfish.Runnable = new(broken)
+func (this *broken) Type() string { return "broken" }
+
+func (this *broken) Run(ctx context.Context) error {
+	log.Println("(i) [broken] wait for approximately 8 minutes")
+	log.Printf("(i) [broken] if impatient, run: kill -9 %d", os.Getpid())
+	<- (chan struct { })(nil)
+	return ctx.Err() // unreachable, of course
+}

examples/http/main.go

@@ -0,0 +1,90 @@
+// Example http demonstrates the usage of [camfish.RunShutdowner] to run an http
+// server.
+package main
+
+import "fmt"
+import "log"
+import "iter"
+import "errors"
+import "context"
+import "net/http"
+import "git.tebibyte.media/sashakoshka/camfish"
+import "git.tebibyte.media/sashakoshka/go-util/sync"
+
+func main() {
+	camfish.Run("http",
+		"Example http demonstrates the usage of " +
+		"camfish.RunShutdowner to run an http server",
+		new(httpServer),
+		new(database))
+}
+
+// httpServer serves data over http.
+type httpServer struct {
+	server *http.Server
+	database *database
+}
+var _ camfish.RunShutdownable = new(httpServer)
+var _ camfish.Initializable = new(httpServer)
+func (this *httpServer) Type() string { return "http-server" }
+
+func (this *httpServer) Init(ctx context.Context) error {
+	this.server = &http.Server {
+		Addr: "localhost:8080",
+		Handler: this,
+	}
+	if actor, ok := camfish.Find("database").(*database); ok {
+		this.database = actor
+	} else {
+		return errors.New("could not locate database")
+	}
+	return ctx.Err()
+}
+
+func (this *httpServer) Run() error {
+	log.Printf("(i) [http-server] listening on %s", this.server.Addr)
+	err := this.server.ListenAndServe()
+	if errors.Is(err, http.ErrServerClosed) { return nil }
+	return err
+}
+
+func (this *httpServer) Shutdown(ctx context.Context) error {
+	return this.server.Shutdown(ctx)
+}
+
+func (this *httpServer) ServeHTTP(res http.ResponseWriter, req *http.Request) {
+	fmt.Fprintf(res, "<!DOCTYPE html><html><head><title>inventory</title></head><body>")
+	fmt.Fprintf(res, "<table><tr><th>Item</th><th>Count</th></tr>")
+	for item, count := range this.database.Inventory() {
+		fmt.Fprintf(res, "<tr><td>%s</td><td>%d</td></tr>", item, count)
+	}
+	fmt.Fprintf(res, "</table>")
+	fmt.Fprintf(res, "</body></html>")
+}
+
+// database provides data that can be served.
+type database struct {
+	inventory usync.RWMonitor[map[string] int]
+}
+func (this *database) Type() string { return "database" }
+
+func (this *database) Init(ctx context.Context) error {
+	this.inventory.Set(map[string] int {
+		"screws": 34,
+		"blood": 90,
+		"paperclips": 5230,
+		"wood": 3,
+		"grains of rice": 238409,
+	})
+	return ctx.Err()
+}
+
+func (this *database) Inventory() iter.Seq2[string, int] {
+	return func(yield func(string, int) bool) {
+		inventory, done := this.inventory.RBorrow()
+		defer done()
+		for item, amount := range inventory {
+			yield(item, amount)
+		}
+	}
+}

examples/panic/main.go

@@ -0,0 +1,35 @@
+// Example panic demonstrates how the environment can restart actors if they
+// fail.
+package main
+
+import "log"
+import "time"
+import "errors"
+import "context"
+import "math/rand"
+import "git.tebibyte.media/sashakoshka/camfish"
+
+func main() {
+	camfish.Run("panic",
+		"Example panic demonstrates how the environment can restart " +
+		"actors if they fail",
+		new(actor))
+}
+
+// actor is an incorrectly implemented actor that panics and errs randomly.
+type actor struct { }
+var _ camfish.Runnable = new(actor)
+func (this *actor) Type() string { return "panic" }
+
+func (this *actor) Run(ctx context.Context) error {
+	log.Println("(i) [panic] panicking in 10 seconds")
+	select {
+	case <- ctx.Done(): return ctx.Err()
+	case <- time.After(time.Second * 10):
+		if rand.Int() % 2 == 0 {
+			panic("this is a panic")
+		} else {
+			return errors.New("this is an error")
+		}
+	}
+}

ini.go

@@ -162,7 +162,7 @@ func configFiles(program string) ([]string, error) {
 	userConfig, err := os.UserConfigDir()
 	if err != nil { return nil, err }
 	return []string {
-		filepath.Join("/etc", program),
+		filepath.Join("/etc", program, program + ".conf"),
-		filepath.Join(userConfig, program),
+		filepath.Join(userConfig, program, program + ".conf"),
 	}, nil
 }

mock/config_test.go

@@ -8,16 +8,16 @@ func TestConfig(test *testing.T) {
 		"multiple": []string { "item0", "item1" },
 		"empty": []string { },
 	}
-	if correct, got := config.Get("single"), "aslkdjasd"; correct != got {
+	if correct, got := "aslkdjasd", config.Get("single"); correct != got {
 		test.Fatal("not equal:", got)
 	}
-	if correct, got := config.Get("multiple"), "item0"; correct != got {
+	if correct, got := "item0", config.Get("multiple"); correct != got {
 		test.Fatal("not equal:", got)
 	}
-	if correct, got := config.Get("empty"), ""; correct != got {
+	if correct, got := "", config.Get("empty"); correct != got {
 		test.Fatal("not equal:", got)
 	}
-	if correct, got := config.Get("non-existent"), ""; correct != got {
+	if correct, got := "", config.Get("non-existent"); correct != got {
 		test.Fatal("not equal:", got)
 	}
 	for index, value := range config.GetAll("single") {

phases.go

@@ -6,6 +6,7 @@ import "log"
 import "io/fs"
 import "errors"
 import "context"
+import "runtime"
 import "strings"
 import "path/filepath"
 import "git.tebibyte.media/sashakoshka/go-cli"
@@ -18,12 +19,15 @@ func (this *environment) phase10FlagParsing() bool {
 		name: this.name,
 		description: this.description,
 	}
-	flagHelp          := set.Flag('h', "help",          "Display usage information and exit", nil)
+	flagHelp          := set.Flag('h', "help",           "Display usage information and exit", nil)
-	flagPidFile       := set.Flag('p', "pid-file",      "Write the PID to the specified file", cli.ValString)
+	flagPidFile       := set.Flag('p', "pid-file",       "Write the PID to the specified file", cli.ValString)
-	flagUser          := set.Flag('u', "user",          "The user:group to run as", cli.ValString)
+	flagUser          := set.Flag('u', "user",           "The user:group to run as", cli.ValString)
-	flagLogDirectory  := set.Flag('l', "log-directory", "Write logs to the specified directory", cli.ValString)
+	flagLogDirectory  := set.Flag('l', "log-directory",  "Write logs to the specified directory", cli.ValString)
-	flagConfigFile    := set.Flag('c', "config-file",   "Use this configuration file", cli.ValString)
+	flagConfigFile    := set.Flag('c', "config-file",    "Use this configuration file", cli.ValString)
-	flagVerbose       := set.Flag('v', "verbose",       "Enable verbose output/logging", nil)
+	flagVerbose       := set.Flag('v', "verbose",        "(debug) Enable verbose output/logging", nil)
+	flagCrash         := set.Flag(0,   "crash",          "(debug) Crash when an actor panics", nil)
+	flagCrashOnError  := set.Flag(0,   "crash-on-error", "(debug) Crash when an actor experiences any error", nil)
+	flagFastTiming    := set.Flag(0,   "fast-timing",    "(debug) Make timed things happen faster/more often", nil)
 
 	// ask actors to add flags
 	actors, done := this.actors.RBorrow()
@@ -60,6 +64,17 @@ func (this *environment) phase10FlagParsing() bool {
 	if _, ok := flagVerbose.First(); ok {
 		this.flags.verbose = true
 	}
+	if _, ok := flagCrash.First(); ok {
+		this.flags.crash = true
+	}
+	if _, ok := flagCrashOnError.First(); ok {
+		this.flags.crash = true
+		this.flags.crashOnError = true
+	}
+	if _, ok := flagFastTiming.First(); ok {
+		this.flags.fastTiming = true
+		this.cron.fastTiming = true
+	}
 	return true
 }
 
@@ -128,6 +143,12 @@ func (this *environment) phase30ConfigurationParsing() bool {
 	if this.flags.configFile != "" {
 		paths = append(paths, this.flags.configFile)
 	}
+	if this.Verb() {
+		log.Println("(i) (30) have configuration files:")
+		for _, paths := range paths {
+			log.Println("(i) (30) -", paths)
+		}
+	}
 	// parse every config and merge them all
 	configs := make([]iniConfig, 0, len(paths))
 	for _, path := range paths {
@@ -199,7 +220,7 @@ func (this *environment) phase60Initialization() bool {
 		initializable = actors.initializable.all()
 	}()
 	if err := this.initializeActors(this.ctx, initializable...); err != nil {
-		log.Println(".// (60) failed to initialize:", err)
+		log.Println("XXX (60) failed to initialize:", err)
 		return false
 	}
 	if this.Verb() { log.Println(".// (60) initialized") }
@@ -215,7 +236,10 @@ func (this *environment) phase70Running() bool {
 		actors, done := this.actors.RBorrow()
 		defer done()
 		for _, actor := range actors.runnable.all() {
-			this.start(actor)
+			this.start(actor.(Actor))
+		}
+		for _, actor := range actors.runShutdownable.all() {
+			this.start(actor.(Actor))
 		}
 		
 	}()
@@ -245,6 +269,9 @@ func (this *environment) phase70_5Trimming() bool {
 	}()
 	if err := this.trimActors(this.ctx, trimmable...); err != nil {
 		log.Println(".// (70.5) failed to trim:", err)
+		if this.flags.crashOnError {
+			panic(err)
+		}
 		return false
 	}
 	if this.Verb() { log.Println(".// (70.5) trimmed") }
@@ -252,6 +279,25 @@ func (this *environment) phase70_5Trimming() bool {
 }
 
 func (this *environment) phase80Shutdown() bool {
+	logActors(All())
+	ctx, done := context.WithTimeout(
+		context.Background(),
+		defaul(this.timing.shutdownTimeout.Load(), defaultShutdownTimeout))
+	defer done()
+	go func() {
+		<- ctx.Done()
+		if errors.Is(context.Cause(ctx), context.DeadlineExceeded) {
+			log.Println("XXX (80) shutdown timeout expired, performing emergency halt")
+			if Verb() || this.flags.crashOnError {
+				dumpBuffer := make([]byte, 8192)
+				runtime.Stack(dumpBuffer, true)
+				log.Printf("XXX (80) stack trace of all goroutines:\n%s", dumpBuffer)
+			}
+			log.Printf("====== [%s] END =======", this.name)
+			os.Exit(1)
+		}
+	}()
+	
 	cause := context.Cause(this.ctx)
 	if cause != nil {
 		log.Println("XXX (80) shutting down because:", cause)

run.go

@@ -9,7 +9,7 @@ var env environment
 // when all running actors have stopped. Error and log messages will be printed.
 // The correct way to use this function is to have it be the only thing in main:
 // 
-//   func main () {
+//   func main() {
 //           camfish.Run("name", "what it does", new(SomeActor), new(AnotherActor))
 //   }
 //
@@ -50,21 +50,22 @@ var env environment
 //       is configurable, but by default it is 8 minutes. The vast majority of
 //       actors should initialize in under 100 milliseconds.
 //   
-//   70. Running: Actors which implement [Runnable] are run, each in their own
+//   70. Running: Actors which implement [Runnable] or [RunShutdownable] are
-//       goroutine. The environment is able to restart actors which have failed,
+//       run, each in their own goroutine. The environment is able to restart
-//       which entails resetting the actor if it implements [Resettable], and
+//       actors which have failed, which entails resetting the actor if it
-//       running the actor again within the same goroutine. If an actor does not
+//       implements [Resettable], and running the actor again within the same
-//       run for a meaningful amount of time after resetting/initialization
+//       goroutine. If an actor does not run for a meaningful amount of time
-//      before failing, it is considered erratic and further attempts to restart
+//       after resetting/initialization before failing, it is considered erratic
-//      it will be spaced by a limited, constantly increasing time interval. The
+//       and further attempts to restart it will be spaced by a limited,
-//      timing is configurable, but by default the threshold for a meaningful
+//       constantly increasing time interval. The timing is configurable, but by
-//      amount of runtime is 16 seconds, the initial delay interval is 8
+//       default the threshold for a meaningful amount of runtime is 16 seconds,
-//      seconds, the interval increase per attempt is 8 seconds, and the maximum
+//       the initial delay interval is 8 seconds, the interval increase per
-//      interval is one hour. Additionally, programs which implement [Trimmable]
+//       attempt is 8 seconds, and the maximum interval is one hour.
-//      will be trimmed regularly whenever they are running. The trimming
+//       Additionally, programs which implement [Trimmable] will be trimmed
-//      interval is also configurable, but by default it is once every minute.
+//       regularly whenever they are running. The trimming interval is also
-//      When an actor which implements [Resettable] is reset, it is given a
+//       configurable, but by default it is once every minute. When an actor
-//      configurable timeout, which is 8 minutes by default.
+//       which implements [Resettable] is reset, it is given a configurable
+//       timeout, which is 8 minutes by default.
 //   
 //   80. Shutdown: This can be triggered by all actors being removed from the
 //       environment, a catastrophic error, [Done] being called, or the program

util.go

@@ -12,6 +12,19 @@ import "strings"
 import "context"
 import "sync/atomic"
 import "unicode/utf8"
+import "runtime/debug"
+
+func panicErr(message any, stack []byte) (err error) {
+	if panErr, ok := message.(error); ok {
+		err = panErr
+	} else {
+		err = errors.New(fmt.Sprint(message))
+	}
+	if stack != nil {
+		err = fmt.Errorf("%w: %s", err, stack)
+	}
+	return err
+}
 
 func defaul[T comparable](value, def T) T {
 	var zero T
@@ -19,14 +32,21 @@ func defaul[T comparable](value, def T) T {
 	return value
 }
 
-func panicWrap(ctx context.Context, f func (context.Context) error) (err error) {
+func panicWrap(f func() error) (err error) {
 	defer func () {
 		if pan := recover(); pan != nil {
-			if panErr, ok := pan.(error); ok {
+			err = panicErr(pan, debug.Stack())
-				err = panErr
+		}
-			} else {
+	} ()
-				err = errors.New(fmt.Sprint(pan))
+	
-			}
+	err = f()
+	return
+}
+
+func panicWrapCtx(ctx context.Context, f func(context.Context) error) (err error) {
+	defer func () {
+		if pan := recover(); pan != nil {
+			err = panicErr(pan, debug.Stack())
 		}
 	} ()
 	
@@ -62,7 +82,11 @@ func logActors (actors iter.Seq[Actor]) {
 	}
 	types := make(map[string] int)
 	for actor := range actors {
-		types[actor.Type()] += 1
+		typ := actor.Type()
+		if named, ok := actor.(Named); ok {
+			typ = fmt.Sprintf("%s/%s", typ, named.Name())
+		}
+		types[typ] += 1
 	}
 	for typ, count := range types {
 		if count > 1 {

util_test.go

@@ -17,17 +17,35 @@ func TestDefaul(test *testing.T) {
 }
 
 func TestPanicWrap(test *testing.T) {
-	err := panicWrap(context.Background(), func (ctx context.Context) error {
+	err := panicWrap(func () error {
 		return errors.New("test case 0")
 	})
 	test.Log(err)
 	if err.Error() != "test case 0" { test.Fatal("not equal") }
-	err = panicWrap(context.Background(), func (ctx context.Context) error {
+	err = panicWrap(func () error {
 		panic(errors.New("test case 1"))
 	})
 	test.Log(err)
 	if err.Error() != "test case 1" { test.Fatal("not equal") }
-	err = panicWrap(context.Background(), func (ctx context.Context) error {
+	err = panicWrap( func () error {
+		return nil
+	})
+	test.Log(err)
+	if err != nil { test.Fatal("not equal") }
+}
+
+func TestPanicWrapCtx(test *testing.T) {
+	err := panicWrapCtx(context.Background(), func (ctx context.Context) error {
+		return errors.New("test case 0")
+	})
+	test.Log(err)
+	if err.Error() != "test case 0" { test.Fatal("not equal") }
+	err = panicWrapCtx(context.Background(), func (ctx context.Context) error {
+		panic(errors.New("test case 1"))
+	})
+	test.Log(err)
+	if err.Error() != "test case 1" { test.Fatal("not equal") }
+	err = panicWrapCtx(context.Background(), func (ctx context.Context) error {
 		return nil
 	})
 	test.Log(err)