#!/usr/bin/env python
"""
Build a single AnnData object with spliced/unspliced count layers and sample
metadata from split-pipe sublibrary outputs, ready to hand off to scVelo.

Example:
    python generate_scvelo_adata.py /volume-general/analysis/tutorials/scvelo/ \
        --comb-dir comb-out --out /volume-general/analysis/tutorials/scvelo/scvelo_adata_raw.h5ad
"""

import argparse
import json
import os
import subprocess
from importlib.metadata import PackageNotFoundError, version

import anndata as ad
import dask.dataframe as dd
import pandas as pd
import scipy.sparse
import scvelo as scv
from dask.diagnostics import ProgressBar
from packaging.version import Version

ProgressBar().register()

# anndata >= 0.13 refuses to write string columns (e.g. obs/var indices, the
# "barcodes" column below) unless this is set, raising:
#   RuntimeError: `anndata.settings.allow_write_nullable_strings` is None and
#   `pd.options.future.infer_string` is False.
# See the "Miscellaneous changes" note in the anndata 0.13.0 release notes.
ad.settings.allow_write_nullable_strings = True

# Known-bad combinations, checked at runtime rather than pinned in requirements
# since this script is often run in a shared/pre-existing environment we don't
# control. Update these as upstream fixes land:
#   - scvelo <= 0.3.4 (latest as of 2026-08) breaks under pandas >= 3: its
#     recover_dynamics() passes a list to pandas.unique(), which pandas 3 no
#     longer accepts (https://github.com/theislab/scvelo/issues/1344).
#     anndata/scanpy have added explicit pandas 3 support, but ad.concat()
#     (used below in build_adata) still has open edge cases on pandas >= 3
#     (https://github.com/scverse/anndata/issues/2475).
#   - scvelo <= 0.3.4 also has an open numpy 2.x bug in leastsq_generalized()
#     (https://github.com/theislab/scvelo/issues/1341).
_PANDAS_3_INCOMPATIBLE_SCVELO = Version("0.3.4")
_NUMPY_2_INCOMPATIBLE_SCVELO = Version("0.3.4")


def check_package_versions():
    """Warn (without raising) about package combinations known to conflict."""
    versions = {}
    for pkg in ("pandas", "numpy", "anndata", "scanpy", "scvelo", "dask"):
        try:
            versions[pkg] = Version(version(pkg))
        except PackageNotFoundError:
            print(f"WARNING: could not determine installed version of {pkg}.")

    warnings = []
    if versions.get("pandas", Version("0")) >= Version("3") and versions.get(
        "scvelo", Version("0")
    ) <= _PANDAS_3_INCOMPATIBLE_SCVELO:
        warnings.append(
            f"pandas {versions['pandas']} is installed alongside scvelo "
            f"{versions['scvelo']}, which is known to break under pandas >= 3 "
            "(recover_dynamics() calls pandas.unique() on a list, which pandas 3 "
            "no longer accepts: https://github.com/theislab/scvelo/issues/1344). "
            "If scVelo calls on the AnnData this script produces fail, try "
            "`pip install \"pandas<3\"`."
        )
    if versions.get("numpy", Version("0")) >= Version("2") and versions.get(
        "scvelo", Version("0")
    ) <= _NUMPY_2_INCOMPATIBLE_SCVELO:
        warnings.append(
            f"numpy {versions['numpy']} is installed alongside scvelo "
            f"{versions['scvelo']}, which has an open bug under numpy 2.x in "
            "leastsq_generalized() (https://github.com/theislab/scvelo/issues/1341)."
        )

    if warnings:
        print("WARNING: potentially conflicting package versions detected:")
        for w in warnings:
            print(f"  - {w}")


def generate_splice_matrices(tscp_path, cell_barcodes):
    """
    Build spliced/unspliced matrices for the barcodes split-pipe called as cells.

    cell_barcodes are this sublibrary's bc_wells values (without the "__sN"
    suffix), taken from the combined cell_metadata.csv. We use split-pipe's own
    cell calls rather than recomputing a transcript cutoff, because split-pipe
    calls cells per sample with a separate cutoff for each; the all-sample
    cell_tscp_cutoff in agg_sample_summary.csv is only a summary value, and
    applying it to every barcode admits/drops cells near each sample's threshold.
    """
    print(f"Reading in {tscp_path}")
    # gene_name can be blank for many rows (e.g. Mmul10 genes without symbols), which
    # makes dask infer float64 from the first block; force string dtypes instead
    tscp_assign_df = dd.read_csv(
        tscp_path, blocksize="800MB", dtype={"gene": "object", "gene_name": "object"}
    ).compute()
    # fall back to the gene ID so unnamed genes aren't collapsed into a single NaN gene
    tscp_assign_df["gene_name"] = tscp_assign_df["gene_name"].fillna(tscp_assign_df["gene"])

    cell_barcodes = pd.Index(cell_barcodes)
    observed = pd.Index(tscp_assign_df["bc_wells"].unique())
    bcs = cell_barcodes[cell_barcodes.isin(observed)]
    n_missing = len(cell_barcodes) - len(bcs)
    if n_missing:
        print(
            f"WARNING: {n_missing} of {len(cell_barcodes)} called cells for this "
            "sublibrary have no transcripts in the transcript assignment file and "
            "will be left out."
        )
    if len(bcs) == 0:
        raise ValueError(
            f"None of the called cells for this sublibrary were found in {tscp_path}. "
            "Check that the sublibrary order in run_proc_def.json matches this data."
        )

    genes = tscp_assign_df.gene_name.unique()
    gene_dict = dict(zip(genes, range(len(genes))))
    barcode_dict = dict(zip(bcs, range(len(bcs))))

    print("Filtering tscp file..")
    tscp_assign_df_filt = tscp_assign_df[tscp_assign_df.bc_wells.isin(barcode_dict)].copy()
    tscp_assign_df_filt["cell_index"] = tscp_assign_df_filt.bc_wells.map(barcode_dict)
    tscp_assign_df_filt["gene_index"] = tscp_assign_df_filt.gene_name.map(gene_dict)
    print("Done:", tscp_assign_df_filt.shape)

    rcv = tscp_assign_df_filt.query("exonic").groupby(["cell_index", "gene_index"]).size().reset_index().values
    rows = list(rcv[:, 0]) + [len(barcode_dict) - 1]
    cols = list(rcv[:, 1]) + [len(genes) - 1]
    vals = list(rcv[:, 2]) + [0]
    X_exonic = scipy.sparse.csr_matrix((vals, (rows, cols)))

    rcv = tscp_assign_df_filt.query("~exonic").groupby(["cell_index", "gene_index"]).size().reset_index().values
    rows = list(rcv[:, 0]) + [len(barcode_dict) - 1]
    cols = list(rcv[:, 1]) + [len(genes) - 1]
    vals = list(rcv[:, 2]) + [0]
    X_intronic = scipy.sparse.csr_matrix((vals, (rows, cols)))

    adata = scv.AnnData(X=X_exonic + X_intronic)
    adata.obs = pd.DataFrame({"barcodes": bcs}, index=bcs.astype(str))
    adata.var = pd.DataFrame({"gene": genes, "gene_name": genes}, index=genes.astype(str))
    adata.var_names_make_unique()
    adata.obs_names_make_unique()
    adata.layers["spliced"] = X_exonic
    adata.layers["unspliced"] = X_intronic
    scv.utils.show_proportions(adata)
    return adata


def get_sublib_names(comb_dir):
    """
    Determine each sublibrary's folder name and its "__sN" identifier directly
    from the combine-mode output's run_proc_def.json, rather than assuming a
    fixed naming convention (e.g. "S1-out") for sublibrary folders. The
    "all-sample" entry's "sublib_list" records both the folder name split-pipe
    was given for each sublibrary and the order they were combined in
    (index 0 -> __s1, index 1 -> __s2, ...), which is what actually determines
    the barcode suffixes used in the combined metadata.
    """
    proc_def_path = f"{comb_dir}/process/run_proc_def.json"
    with open(proc_def_path) as f:
        run_def = json.load(f)

    all_sample = next(s for s in run_def["sample"]["samples"] if s["name"] == "all-sample")
    return [entry["sublib_name"] for entry in all_sample["sublib_list"]]


def build_adata(working_dir, comb_dir="comb-out", out_dir=None):
    working_dir = working_dir.rstrip("/") + "/"
    # Default to writing alongside the input data, but let callers (e.g. main(),
    # via --out) redirect all output — including the per-sublibrary intermediates
    # below — away from working_dir, since that may be a shared/read-only dataset
    # directory rather than a place we want to leave generated files.
    out_dir = (out_dir or working_dir).rstrip("/") + "/"

    comb_dir_path = f"{working_dir}{comb_dir}"
    sublib_names = get_sublib_names(comb_dir_path)
    if not sublib_names:
        raise ValueError(f"No sublibraries listed in {comb_dir_path}/process/run_proc_def.json")

    meta_path = f"{comb_dir_path}/all-sample/DGE_filtered/cell_metadata.csv"
    comb_meta = pd.read_csv(meta_path, index_col=0)

    ad_list = []
    for suffix, sublib_name in enumerate(sublib_names, start=1):
        sublib_dir = f"{working_dir}{sublib_name}"
        gz_path = f"{sublib_dir}/process/tscp_assignment.csv.gz"
        unzipped_path = gz_path.replace(".gz", "")
        if not os.path.exists(unzipped_path):
            if not os.path.exists(gz_path):
                raise FileNotFoundError(f"Missing transcript assignment file: {gz_path}")
            # -k keeps the original .gz; check=True raises if gunzip fails
            subprocess.run(["gunzip", "-k", gz_path], check=True)

        # this sublibrary's called cells, from the combined metadata index (bc_wells__sN)
        tag = f"__s{suffix}"
        sublib_cells = comb_meta.index[comb_meta.index.str.endswith(tag)].str.slice(stop=-len(tag))
        if len(sublib_cells) == 0:
            raise ValueError(f"No cells ending in '{tag}' in {meta_path} for sublibrary {sublib_name}")
        adata = generate_splice_matrices(unzipped_path, sublib_cells)

        adata.obs.index = adata.obs.index + f"__s{suffix}"
        adata.write(f"{out_dir}sub{suffix}_adata.h5ad")
        print(f"Saved anndata to {out_dir}sub{suffix}_adata.h5ad.\n")
        ad_list.append(adata)

    ad_splice = ad.concat(ad_list)
    ad_splice.obs = ad_splice.obs.join(comb_meta, how="left")
    return ad_splice


def main():
    check_package_versions()

    parser = argparse.ArgumentParser(
        description="Build a combined spliced/unspliced AnnData object from split-pipe "
        "sublibrary outputs, for use with scVelo."
    )
    parser.add_argument(
        "working_dir",
        help="Directory containing the sublibrary output folders and the combine-mode output folder",
    )
    parser.add_argument(
        "--comb-dir",
        default="comb-out",
        help="Name of the split-pipe combine-mode output folder within working_dir "
        "(default: comb-out)",
    )
    parser.add_argument(
        "--out",
        default=None,
        help="Path to the combined .h5ad file, or a folder to write scvelo_adata_raw.h5ad "
        "into. Per-sublibrary sub{N}_adata.h5ad files are written to the same folder "
        "(default: <working_dir>/scvelo_adata_raw.h5ad)",
    )
    args = parser.parse_args()

    default_name = "scvelo_adata_raw.h5ad"
    if args.out is None:
        out_path = os.path.join(args.working_dir, default_name)
    elif args.out.endswith(".h5ad"):
        out_path = args.out
    else:
        # --out names a folder (existing or not, with or without a trailing slash)
        out_path = os.path.join(args.out, default_name)
    out_dir = os.path.dirname(out_path) or "."
    os.makedirs(out_dir, exist_ok=True)

    ad_splice = build_adata(args.working_dir, args.comb_dir, out_dir=out_dir)

    ad_splice.write(out_path)
    print(f"Saved combined anndata to {out_path}")


if __name__ == "__main__":
    main()
