Skip to main content

  • `SELECT COUNT(id) from my_big_table` could be too slow for huge tables since it traverses all rows. Instead, you can get an approximation that's updated every AUTOVACUUM.
    defmodule MyApp.Repo do
      use Ecto.Repo,
        otp_app: :my_app,
        adapter: Ecto.Adapters.Postgres
    
      @doc """
      # Fast Postgres counting.
    
      This will query internal PG tables that're updated by AUTOVACUUM. It's an approximate count based on the total disk
      size of the table divided by the average size of row.
    
      For example:
        ACTUAL count `SELECT count(id) from my_big_table` = 4535133
        APPROXIMATE count 4499917 (off by 35216)
    
      Crash course:
    
        reltuples = # of rows (relation) on a page
        relpages = # of pages of rows
        These ^ are updated based on the last AUTOVACUUM which runs periodically
    
        Get the average # of rows on a page
          eg: reltuples = 4499917, relpages = 2050761
              4499917 / 2050761 = 2.1942669087232
    
        Postgres essentially stores a page on a disk block.
        So now we need to query disk information:
    
        pg_relation_size = disk size of table
        current_setting('block_size') = disk block size
    
        Get the row size per block for the table
          eg: pg_relation_size = 16799834112, block_size = 8192
              16799834112 / 8192 = 2050761
    
        Now multiply the rows per page with the row size per block
          eg: 2.1942669087232 * 2050761 = 4499917
      """
      def fast_table_count(ecto_schema) do
        table = ecto_schema.__schema__(:source)
        case Ecto.Adapters.SQL.query!(
               __MODULE__,
               """
               SELECT
                 (NULLIF(reltuples, 0) / NULLIF(relpages, 0)) * (
                  pg_relation_size('#{table}') / (
                  current_setting('block_size')::INTEGER)
                 )
               FROM pg_class
               WHERE relname = '#{table}'
               """
             ) do
          %{rows: [[nil]]} -> 0
          %{rows: [[count]]} -> count
        end
      end
    end
    28 upvotes
  • Isaac Yonemoto

    ityonemo

    makes logging sane.

    put this in your .iex.exs; and log lines will bump the console prompt down.
    if function_exported?(Mix, :__info__, 1) and Mix.env() == :dev do
      # if statement guards you from running it in prod, which could result in loss of logs.
      Logger.configure_backend(:console, device: Process.group_leader())
    end
    
    11 upvotes
  • David Bernheisel

    dbernheisel

    bernheisel

    Disable FLoC in Phoenix

    Want to disable Federated Learning of Cohorts (FLoC) in Phoenix Framework? https://web.dev/floc/ h/t to @mcrumm
    # my_app_web/router.ex
    
    defmodule MyAppWeb.Router do
      use MyAppWeb, :router
    
      pipeline :browser do
        # ...other plugs...
    
    
        plug :put_secure_browser_headers, %{"permissions-policy" => "interest-cohort=()"}
      end
    end
    9 upvotes
  • Stream.resource is great for indeterminate streams, like API pages. Build a request, then handle the next page until it's finished
    def stargazers(%CodeRepo{owner: owner, name: name}) do
      Stream.resource(
        fn -> 
          Finch.build(:get, @base_url <> "/repos/#{owner}/#{repo}/stargazers", @default_headers)
        end,
        fn
          nil ->
            {:halt, nil}
    
          request ->
            handle_request(:get, request)
        end,
        fn _request -> nil end
      )
    end
    
    defp handle_request(verb, request) do
      case Finch.request(request, HTTP) do
        {:ok, %{body: body, status: status} = response} when status in @success ->
          {[response], next_page_request(verb, response)}
    
        {:ok, %{body: body, status: 429} = response} ->
          handle_rate_limit(verb, response)
    
        {_, response} ->
          {[response], nil}
      end
    end
    21 upvotes
  • When doing REPL-driven development, either on iex or on erl, the commands below are the ones I type when the screen is full of past commands and I want to clear the screen and go to the top.
    iex> clear
    erl> io:format("\ec").
    15 upvotes
  • In Elixir, sorting occurs between types as: number < atom < reference < function < port < pid < tuple < map < list < bitstring. So be sure that your value is not nil or any other atom.
    nil >= 3
    # => true
    
    
    # Because the following is true
    
    is_atom(nil)
    # => true 
    
    
    # Which also leads to that funny little truth:
    
    :negative_infinitiy > 42
    # => true
    
    
    # Also:
    
    Integer.parse("fortytwo") > 100
    # => true
    
    # => Integer.parse("fortytwo") results in :error
    9 upvotes
  • Jason Axelson

    axelson

    Generate a unique positive integer

    Generates an increasing integer, that is unique to this BEAM instance.
    iex> System.unique_integer([:positive, :monotonic])
    1
    iex> System.unique_integer([:positive, :monotonic])
    2
    iex> System.unique_integer([:positive, :monotonic])
    3
    15 upvotes
  • You can validate a field based on if it's in memory or persisted. Use Ecto.get_meta/2 to check for the state of the struct https://hexdocs.pm/ecto/Ecto.Schema.Metadata.html h/t to @sleeplessgeek
    import Ecto.Changeset
    
    def ensure_hostname_is_permanent(changeset) do
      state = Ecto.get_meta(changeset, :state)
    
      # state will be :built (in memory), :loaded (persisted), or :deleted (on its way out)
    
      if get_change(changeset, :hostname) && state != :built do
        add_error(changeset, :hostname, "Cannot change hostname after persisting")
      else
        changeset
      end
    end
    29 upvotes
  • Isaac Yonemoto

    ityonemo

    IO.inspect snippet for VSCode

    Makes multiline IO.inspects easy. You can usually ninja them out easily as a multiline too.
    {
    	"Inspect": {
    		"prefix": "ins",
    		"body": "|> IO.inspect(label: \"$0$TM_LINE_NUMBER\")",
    		"description": "Adds a pipeline with a labelled `IO.inspect`",
    	}
    }
    
    14 upvotes
  • UTF-32 and UTF-16 binary can contain byte-order-marks, which complicate translating to ASCII or UTF-8. Here's how to trim the BOM off the binary.
    # Construct our UTF-32 string with a BOM
    
    iex> utf32_with_bom = <<0x00, 0x00, 0xFE, 0xFF>> <> :unicode.characters_to_binary("foo", :utf8, :utf32)
    <<0, 0, 254, 255, 0, 0, 0, 102, 0, 0, 0, 111, 0, 0, 0, 111>>
    
    # Convert it to UTF-8, see the BOM?
    
    iex> :unicode.characters_to_binary(utf32_with_bom, :utf32, :utf8)
    "\uFEFFfoo"
    
    # Try to convert to ASCII. Notice the error
    
    iex> :unicode.characters_to_binary(utf32_with_bom, :utf32, :latin1)                                    
    {:error, "", <<0, 0, 254, 255, 0, 0, 0, 102, 0, 0, 0, 111, 0, 0, 0, 111>>}
    
    # Get the BOM byte size
    
    iex> {_encoding, bom} = :unicode.bom_to_encoding(utf32_with_bom)                                       
    {{:utf32, :big}, 4}
    
    # Pattern-match the BOM and trimmed UTF-32 binary
    
    iex> <<_skip::bytes-size(bom), utf32::binary>> = utf32_with_bom                                        
    <<0, 0, 254, 255, 0, 0, 0, 102, 0, 0, 0, 111, 0, 0, 0, 111>>
    
    # Now convert to ASCII
    
    iex> :unicode.characters_to_binary(utf32, :utf32, :latin1)                                             
    "foo"
    11 upvotes
  • Postgres can perform quick full-text search across columns with the help of generated columns.
    # Setup the migration
    
    execute """
            ALTER TABLE tips
            ADD COLUMN searchable tsvector
            GENERATED ALWAYS AS (
              to_tsvector('english', coalesce(title, '') || ' ' || coalesce(description, ''))
            ) STORED
            """,
            "ALTER TABLE tips DROP COLUMN searchable"
             
    create index("tips", ["searchable"],
             name: :tips_searchable_index,
             using: "GIN",
             concurrently: true
           )
    
    # Use it in your queries
    
    import Ecto.Query 
    
    def search(queryable \\ Tip, search_terms) do
      queryable
      |> where(
        [q],
        fragment("? @@ websearch_to_tsquery('english', ?)", q.searchable, ^search_terms)
      )
      |> order_by([q], [
        asc: fragment(
          "ts_rank_cd(?, websearch_to_tsquery('english', ?), 4)", q.searchable, ^search_terms),
        desc: q.published_at
      ])
    end
    55 upvotes
  • To serve Phoenix apps without a reverse proxy like Nginx or Apache, the OS must allow the Erlang VM to listen in port 80, and 443 if you want https. By default you can't.
    # Replace <path_to_erlang_folders> with the path to the VM that will run your app 
    
    # and run this on the command line in your server, as root.
    
    
    setcap 'cap_net_bind_service=+ep' <path_to_erlang_folders>/erts-11.1/bin/beam.smp
    7 upvotes
  • This is a common recurring lookup at my company, how to convert from a map to a struct and vice versa
    defmodule BestStructEver do
      defstruct [:a]
    end
    
    # Struct to simple map:
    
    iex> Map.from_struct(%BestStructEver{a: "foobar"})
    # => %{a: "foobar"}
    
    
    # Map to a struct:
    
    iex> struct(BestStructEver, %{a: "foobar"})
    # => %BestStructEver{a: "foobar"}
    
    
    # Note: The struct function is from Kernel, so `Kernel.` can be omitted.
    7 upvotes
  • Alexander Koutmos

    akoutmos

    akoutmos

    Simple stream data processing

    The Enum and Stream modules complement each other quite nicely and can be used for easy and effective data processing. Take a look at how you can stream process a CSV file!
    # $ cat ./expenses.csv
    
    # # System76 Lemur Pro,1499.99
    
    # # MacBook Air,1999.99
    
    # # AMD Ryzen 3950x,749.99
    
    # ...
    
    
    iex> "./expenses.csv" |>
    ...> File.stream!() |>
    ...> Stream.map(fn line ->
    ...>   line |>
    ...>   String.trim() |>
    ...>   String.split(",") |>
    ...>   Enum.at(1) |>
    ...>   String.to_float()
    ...> end |>
    ...> Enum.sum()
    4249.97
    
    33 upvotes
  • Alexander Koutmos

    akoutmos

    akoutmos

    ExUnit Test Coverage

    ExUnit is packed with plenty of great develop experience-related goodies. One of my favotites is the built-in coverage reporter!
    $ mix test --cover
    Cover compiling modules ...
    ..
    
    Finished in 0.06 seconds
    1 doctest, 1 test, 0 failures
    
    Randomized with seed 213130
    
    Generating cover results ...
    
    Percentage | Modules
    -----------|---------------------------
       100.00% | CoverageTest
    -----------|---------------------------
       100.00% | Total
    
    Generated HTML coverage results in "cover" directory
    
    40 upvotes
  • Parker Selbert

    sorentwo

    sorentwo

    Oban - Unique Keys

    By default, job uniqueness is based on teh queue, state, and args. Did you know you can restrict checking args to only a subset of keys? https://hexdocs.pm/oban/Oban.html#module-unique-jobs
    # Configure uniqueness only based on the id :id field
    defmodule MyApp.BusinessWorker do
      use Oban.Worker, unique: [keys: [:id]]
    
      # ...
    end
    
    # With an existing job:
    %{id: 1, type: "business", url: "https://example.com"}
    |> MyApp.BusinessWorker.new()
    |> Oban.insert()
    
    # Inserting another job with a different type won't work
    %{id: 1, type: "solo", url: "https://example.com"}
    |> MyApp.BusinessWorker.new()
    |> Oban.insert()
    
    # Inserting another job with a different type won't work
    %{id: 2, type: "business", url: "https://example.com"}
    |> MyApp.BusinessWorker.new()
    |> Oban.insert()
    
    17 upvotes
  • Alexander Koutmos

    akoutmos

    akoutmos

    Tracing GenServer execution

    The BEAM has some of the best observability tools built right into the runtime. Right down to tracing individual GenServer process execution flow!
    iex> {:ok, agent} = Agent.start_link fn -> [] end
    {:ok, #PID<0.112.0>}
    
    iex> :sys.trace(agent, true)
    :ok
    
    iex> Agent.get_and_update(agent, fn state -> {state, [1 | state]} end)
    *DBG* <0.112.0> got call {get_and_update, #Fun<erl_eval.44.12345123} from <0.110.0>
    *DBG* <0.112.0> sent [] to <0.110.0>, new state [1]
    []
    
    iex> Agent.get(agent, fn state -> state end)
    *DBG* <0.112.0> got call {get, #Fun<erl_eval.44.12345124} from <0.110.0>
    *DBG* <0.112.0> sent [1] to <0.110.0>, new state [1]
    [1]
    
    iex> :sys.trace(agent, false)
    :ok
    
    iex> Agent.get(agent, fn state -> state end)
    [1]
    
    129 upvotes
  • Parker Selbert

    sorentwo

    sorentwo

    Oban - Replace Args

    Did you know that you can selectively replace args when inserting a unique job? With `replace_args`, when an existing job matches some unique keys all other args are replaced.
    # Given an existing job with these args:
    %{some_value: 1, other_value: 1, id: 123}
    
    # Attempting to insert a new job with the same `id` key and different values:
    %{some_value: 2, other_value: 2, id: 123}
    |> MyJob.new(schedule_in: 10, replace_args: true, unique: [keys: [:id]])
    |> Oban.isnert()
    
    # Will result in a single job with the args:
    %{some_value: 2, other_value: 2, id:123}
    
    14 upvotes
  • Alexander Koutmos

    akoutmos

    akoutmos

    The string concat operator

    You may know that the <> operator is used to concat binaries (strings)...but did you also know you can use it for pattern matching binaries?
    iex> "You can" <> " " <> "concat binaries!"
    #=> "You can concat binaries!"
    
    iex> case "user:b4c52a55-e2d9-446f-908d-42c9812f2e8a" do
          "admin:" <> id -> {:admin, id}
          "user:" <> id -> {:user, id}
          _ -> {:error, :invalid_format}
        end
    {:user, "b4c52a55-e2d9-446f-908d-42c9812f2e8a"}
    
    46 upvotes
  • Parker Selbert

    sorentwo

    sorentwo

    Oban - Unique jobs

    Did you know that Oban lets you specify constraints to prevent enqueuing duplicate jobs? Uniqueness is enforced as jobs are inserted, dynamically and atomically. https://hexdocs.pm/oban/Oban.html#module-unique-jobs
    # Configure 60 seconds of uniqueness within the worker
    defmodule MyApp.BusinessWorker do
      use Oban.Worker, unique: [period: 60]
    
      # ...
    end
    
    # Manually override the unique period for a single job
    MyApp.BusinessWorker.new(%{id: 1}, unique: [period: 120])
    
    # Override a job to have an infinite unique period, which lasts
    # as long as jobs are persisted
    MyApp.BusinessWorker.new(%{id: 1}, unique: [period: :infinity])
    
    19 upvotes
  • Parker Selbert

    sorentwo

    sorentwo

    Oban - Priority

    Did you know that you can prioritize or de-prioritize jobs in a queue by setting a priority from 0-3? Rather than executing in the order they were scheduled, higher priority jobs execute first. https://hexdocs.pm/oban/Oban.html#module-prioritizing-jobs
    # Configure uniqueness only based on the id :id field
    defmodule MyApp.BusinessWorker do
      use Oban.Worker, queue: :events, priority: 1
    
      # ...
    end
    
    # Manually set a higher priority for a job on the "mega" plan
    MyApp.BusinessWorker.new(%{id: 1, plan: "mega"}, priority: 0)
    
    # Manually set a lower priority for a job on the "free" plan
    MyApp.BusinessWorker.new(%{id: 1, plan: "free"}, priority: 3)
    
    11 upvotes
  • Parker Selbert

    sorentwo

    sorentwo

    Oban - Initially Paused

    In the queue-pause saga, did you know you can start a queue in the paused state? Passing `paused: true` as a queue option prevents the queue from processing jobs when it starts. https://hexdocs.pm/oban/Oban.html#start_link/1-primary-options
    # In a blue-green deployment, it may be necessary to start queues when the node
    # boots yet prevent them from processing jobs until the node is rotated into
    # use.
    config :my_app, Oban,
      queues: [
        mailer: 10,
        alpha: [limit: 10, paused: true],
        gamma: [limit: 10, paused: true],
        omega: [limit: 10, paused: true]
      ],
    ...
    
    # Once the app boots, tell each queue to resume processing:
    for queue <- [:alpha, :gamma, :omega] do
      Oban.resume_queue(queue: queue)
    end
    
    21 upvotes
  • Parker Selbert

    sorentwo

    sorentwo

    Oban - Graceful Shutdown

    Did you know that when an app shuts down Oban pauses all queues and waits for jobs to finish? The time is configurable and defaults to 15 seconds, short enough for most deployments. https://hexdocs.pm/oban/Oban.html#start_link/1-twiddly-options
    # Change the default to 30 seconds
    config :my_app, Oban,
      repo: MyApp.Repo,
      queues: [default: 10]
      shutdown_grace_period: :timer.seconds(30)
    
    # Wait up to an hour for long running jobs in a blue-green style deply
    config :my_app, Oban,
      shutdown_grace_period: :timer.minutes(60)
    
    20 upvotes
  • Parker Selbert

    sorentwo

    sorentwo

    Oban - Pausing Queues

    Did you know that you can pause a queue to stop it from processing more jobs? Calling `pause_queue/2` allows executing jobs to keep running while preventing a queue from fetching new jobs. https://hexdocs.pm/oban/Oban.html#pause_queue/2
    # Pause all instances of the :default queue across all nodes
    Oban.pause_queue(queue: :default)
    
    # Pause only the local instance, leaving instances on any other nodes running
    Oban.pause_queue(queue: :default, local_only: true)
    
    # Queues are namespaced by prefix, so you can pause the :default queue for an
    # isolated supervisor
    Oban.pause_queue(MyApp.A.Oban, queue: :default)
    
    16 upvotes
  • Parker Selbert

    sorentwo

    sorentwo

    Oban - Recording Errors

    Did you know that errors are recorded in the database when a job fails? A job's `errors` field contains a list of the time, attempt and a formatted error message for each failed attempt.
    # Errors look like this, where `at` is a UTC timestamp and `error` is the blamed
    
    # and formatted error message.
    
    [
      %{
        "at" => "2021-02-11T17:01:13.517233Z",
        "attempt" => 1,
        "error" => "** (RuntimeError) Something went wrong!\n..."
      }
    ]
    
    # Check the errors for a job with multiple attempts to see if it failed before
    
    # or it was snoozed.
    
    def perform(%Job{attempt: attempt, errors: errors}) when attempt > 1 do
      case errors do
        [%{"error" => error} | _] ->
          IO.puts "This job failed with the error: " <> error
        [] ->
          IO.puts "This job snoozed, it doesn't have any errors"
      end
    
      :ok
    end
    
    10 upvotes

© 2021 Zest Creative, LLC