Autoscrolling with respect for the user

I couldn't find an implementation of this that didn't have serious flaws, so here's a reliable React hook for ensuring a container scrolls to the bottom as its content grows, except when the user scrolls back to look at something. Returns a `ref` for the container, a `tailRef` for a component at the end of the container's children, and an `autoScroll` function that can be called to manually trigger the autoscroll behavior. Accepts a user scroll callback in case you want to respond to user scrolling events.

import { useCallback, useState, useEffect, useRef } from "react";

// Automatically scroll to the bottom of the container when the size or
// scrolling content changes, unless the user has started scrolling.
// Set onUserScroll to false to always scroll to bottom on component size
// changes, regardless of user activity.
export const useAutoScroll = (onUserScroll = null) => {
  const [container, setContainer] = useState(null);
  const ref = useCallback(node => setContainer(node), []);
  const tailRef = useRef();
  const isUserScrolling = useRef();
  const isAutoScrolling = useRef();
  const lastScrollHeight = useRef();
  const atBottomScrollTop = useRef();
  const isAtBottom = el => Math.abs(el.scrollTop + el.clientHeight - el.scrollHeight) < 1;

  const scrollToBottom = (onComplete = null) => {
    if (!container || isAtBottom(container)) {
      return;
    }
    isAutoScrolling.current = true;
    container.classList.add('auto-scrolling');

    // Check if any ancestor container is user-scrolling or auto-scrolling
    // If so, we need to avoid scrollIntoView which would scroll the ancestor
    const hasScrollingAncestor = container.parentElement?.closest('.user-scrolling, .auto-scrolling') !== null;

    // Use scrollIntoView only if no ancestor is scrolling (nice smooth scroll)
    // Otherwise use scrollTop to avoid triggering ancestor scroll and causing loops
    if (hasScrollingAncestor) {
      // Only scroll this container, don't trigger ancestor scrolling
      container.scrollTop = container.scrollHeight;
      // Immediately mark as complete since scrollTop is synchronous
      atBottomScrollTop.current = container.scrollTop;
      isAutoScrolling.current = false;
      container.classList.remove('auto-scrolling');
      onComplete && onComplete(container);
    } else {
      tailRef.current?.scrollIntoView({ behavior: "smooth" });

      // Check to see if we're at the bottom yet
      setTimeout(() => {
        if (container) {
          if (!isAtBottom(container)) {
            scrollToBottom(onComplete);
          } else {
            atBottomScrollTop.current = container.scrollTop;
            isAutoScrolling.current = false;
            container.classList.remove('auto-scrolling');
            onComplete && onComplete(container);
          }
        }
      }, 50);
    }
  };

  const autoScroll = (force = false, onComplete = null) => {
    if (!isUserScrolling.current || force) {
      scrollToBottom(onComplete);
    }
    else {
      onComplete && onComplete(container);
    }
  };

  const scrollHandler = e => {
    if (e.target !== container) {
      return;
    }
    const hasScrollbar = container.scrollHeight > container.clientHeight;
    lastScrollHeight.current = container.scrollHeight;
    if (atBottomScrollTop.current === null) {
      atBottomScrollTop.current = container.scrollHeight - container.clientHeight;
    }
    const currentScrollTop = container.scrollTop;

    // Clear user scrolling if at bottom or no scrollbar
    if (isAtBottom(container) || !hasScrollbar) {
      atBottomScrollTop.current = currentScrollTop;
      if (isUserScrolling.current) {
        isUserScrolling.current = false;
        container.classList.remove('user-scrolling');
      }
    }
    // Detect user scrolling up
    else if (!isAutoScrolling.current
             && atBottomScrollTop.current != null
             && atBottomScrollTop.current > currentScrollTop) {
      if (!isUserScrolling.current) {
        isUserScrolling.current = true;
        container.classList.add('user-scrolling');
        onUserScroll && onUserScroll(e);
      }
    }
  };

  // Ensure we trigger autoscroll on component content/size change
  useEffect(() => {
    if (container) {
      lastScrollHeight.current = container.scrollHeight;
      container.addEventListener("scroll", scrollHandler);

      const mutationObserver = new MutationObserver((mutations) => {
        // If content was cleared or significantly reduced, reset user scrolling flag
        const contentCleared = container.scrollHeight <= container.clientHeight;
        if (contentCleared) {
          isUserScrolling.current = false;
          container.classList.remove('user-scrolling');
        }

        autoScroll();
      });
      mutationObserver.observe(container, {childList: true, subtree: true, characterData: true});

      let resizeTimeout;
      const resizeObserver = new ResizeObserver((entries) => {
        // Debounce to avoid "ResizeObserver loop completed with undelivered notifications"
        clearTimeout(resizeTimeout);
        resizeTimeout = setTimeout(() => {
          // Clear user scrolling if scrollbar disappears
          const hasScrollbar = container.scrollHeight > container.clientHeight;
          if (!hasScrollbar && isUserScrolling.current) {
            isUserScrolling.current = false;
            container.classList.remove('user-scrolling');
          }

          autoScroll();
        }, 0);
      });
      Array.from(container.children).forEach(child => resizeObserver.observe(child));
      resizeObserver.observe(container);
      // Initialize state
      autoScroll();

      return () => {
        container.removeEventListener("scroll", scrollHandler);
        mutationObserver.disconnect();
        resizeObserver.disconnect();
      };
    }
  }, [container, onUserScroll]);

  return [ref, tailRef, autoScroll];
};

"Cambridge Chronicled" complete

I realized I hadn't posted the completion of this project (about two years ago now), mostly b/c the social media has been managed elsewhere.

The book is available Million Year Picnic in Cambridge, MA or online at Radiator Comics.

Other creative things

Been spending a lot of time on a comic (aka graphic novel memoir for those with upturned noses).

We offered a few chapters at the last MICE, but it's not yet available elsewhere.

It's a bit of humor about raising a kid in Cambridge, but it's also touches a lot of serious topics like racism, diversity, culture, and soul.  It's solely intended to share our unique perspective(s).

It's heartening to see so many around the world joining in protest after four years of xenophobia and hate in the US.  While it's sad that so little has changed since Rodney King (or Emmett Till, for that matter), I hope that more white folk (and by this I mean anyone who has questioned whether people of color are really treated any differently in this country) will take the opportunity to learn more.

Because ultimately, justice in society is everyone's responsibility, not just those with a boot on their neck.

Next time you go to read your favorite blog, pick up some James Baldwin or Ralph Ellison to read instead (you can probably get digital or audio versions of these from your local library for free).

Loading that pickle, what's going on?

tqdm is a handy little utility for showing how well some process is progressing (it works within python code, but you can also use it on any piped shell process).

Pickle is python's built-in serialization, which can take an awful long time on really large objects.  Unfortunately, pickle's only input is an open file.

So here's a little useful snippet for estimating how long it's going to take to deserialize that big python pickle file.

class TQDMBytesReader(object):
    """Show progress while reading from a file"""
    def __init__(self, fd, **kwargs):
        self.fd = fd
        from tqdm import tqdm
        self.tqdm = tqdm(**kwargs)
    def read(self, size=-1):
        bytes = self.fd.read(size)
        self.tqdm.update(len(bytes))
        return bytes
    def readline(self):
        bytes = self.fd.readline()
        self.tqdm.update(len(bytes))
        return bytes
    def __enter__(self):
        self.tqdm.__enter__()
        return self
    def __exit__(self, *args, **kwargs):
        return self.tqdm.__exit__(*args, **kwargs)

with open(filename, "rb") as fd, \
     TQDMBytesReader(fd, desc=f"Loading 'pickle", unit="b",
                     total=os.path.getsize(filename) as reader:
    obj = pickle.load(reader)

Here's the complement, although it's not terribly helpful since pickle serializes everything to memory and only then writes to disk:

class TQDMBytesWriter(object):
    """Show progress while writing to a file"""
    def __init__(self, fd, **kwargs):
        self.fd = fd
        from tqdm import tqdm
        self.tqdm = tqdm(**kwargs)
    def write(self, b):
        bytes_written = self.fd.write(b)
        self.tqdm.update(bytes_written or 0)
        return bytes_written
    def __enter__(self):
        self.tqdm.__enter__()
        return self
    def __exit__(self, *args, **kwargs):
        return self.tqdm.__exit__(*args, **kwargs)

Potamus update

The GAE cost profiling graphs have gotten a facelift, now using flot instead of Google visualizations. I rapidly hit the limit of the GViz capabilities (one notable shortcoming is the lack of support for sparsely-populated data). Most of the controls are now completely client-side, which makes it a lot easier to tweak the graph to get just the information you'd like.

Flot generally provides more CSS-level control over styling, and a nice plugin system to allow for mixing features.

App Engine real-time cost profiling is available on github.  Some assembly required.

Cost profiling on Google App Engine

I've recently been measuring costs for various operations that are currently being performed on Google App Engine.  Google provides some cost estimates on the app engine dashboard, and you can get historical daily totals, but it's generally not straightforward to answer the question "How much does this operation cost (or is going to cost if I ramp up)?".

The google-provided appstats is fine for profiling individual requests, but sometimes you need a much more comprehensive view.

With a Chrome extension to monitor the app engine dashboard numbers, and a small app engine app to collect data, I've managed to collect some interesting intra-day profile data, as well as provide a means for fairly accurate estimates of discrete operations.

Group view (for multiple app IDs).  The artifact on the left is due to two days' worth of missing data.  The lower graph has an obvious daily cron job, while the upper has much more distributed activity:

 Zoomed view (detail for a single app ID).  On this graph, you can see some annotations have been added; the data collector provides an API for applications to post events that can be overlaid on the cost data, making it easy to pick start and end points and calculating the cost for the selected span:

This project is now available on github.  The Chrome extension is based on the OSE (Offline Statistics Estimator) which scrapes usage data and applies customizable usage rates from the GAE dashboard pages.

Enable cProfile in Google App Engine

If it's not readily apparent to you how to enable CPU profiling on Google App Engine (it certainly wasn't to me, aside from a few hand waves at cProfile), this code snippet should get you up and running so you can focus on finding the data you need rather than the implied interfaces you have to mimic.  It uses the standard WSGI middleware hook to wrap an incoming request in a cProfile call, formatting and dumping the resulting stats to the log when the request returns:

def cprofile_wsgi_middleware(app):
    """
    Call this middleware hook to enable cProfile on each request.  Statistics are dumped to
    the log at the end of the request.
    :param app: WSGI app object
    :return: WSGI middleware wrapper
    """
    def _cprofile_wsgi_wrapper(environ, start_response):
        import cProfile, cStringIO, pstats
        profile = cProfile.Profile()
        try:
            return profile.runcall(app, environ, start_response)
        finally:
            stream = cStringIO.StringIO()
            stats = pstats.Stats(profile, stream=stream)
            stats.strip_dirs().sort_stats('cumulative', 'time', 'calls').print_stats(25)
            logging.info('cProfile data:\n%s', stream.getvalue())
    return _cprofile_wsgi_wrapper

def webapp_add_wsgi_middleware(app):
    return cprofile_wsgi_middleware(app)