Hello,
I’m trying to integrate scATAC-seq and scRNA-seq from a 10x Genomics Multiome experiment, but I’m having problems when I run mu.pp.intersect_obs(mdata).
This is the error message:
ValueError: Value passed for key ‘X_pca’ is of incorrect shape. Values of obsm must match dimensions (‘obs’,) of parent. Value had shape (10333,) while it should have had (8988,).
I’m seeing the same error when I try to run mu.pp.filter_obs() in the rna modality.
Thanks,
Leandro
Hey @Drito,
Great timing! This has been due to the recent changes in anndata, and we have just fixed that on our end. Do you think you can check the latest version from the github repo and let us know if it works for you? It didn’t make it to a release just yet.
Thanks for your reply.
I tried it with the new version and I’m still having the same problem.
Best,
Leandro
Hey @Drito,
Thanks for reporting that! Do you think you could provide a minimal example so that we can reproduce that?
Hello,
This is a minimal example of my analysis pipeline:
mdata = mu.read_10x_h5(os.path.join(data_dir, "filtered_feature_bc_matrix.h5"))
mdata.var_names_make_unique()
rna = mdata.mod['rna']
atac = mdata.mod['atac']
# RNA
rna.var['mt'] = rna.var_names.str.startswith("MT-")
sc.pp.calculate_qc_metrics(rna, qc_vars=['mt'], percent_top=None, log1p=False, inplace=True)
mu.pp.filter_var(rna, 'n_cells_by_counts', lambda x: x >= 10)
mu.pp.filter_obs(rna, 'n_genes_by_counts', lambda x: (x >= 500) & (x < 5000))
mu.pp.filter_obs(rna, 'total_counts', lambda x: x < 15000)
mu.pp.filter_obs(rna, 'pct_counts_mt', lambda x: x < 20)
sc.pp.scrublet(rna)
doublets = sum(rna.obs.predicted_doublet)
mu.pp.filter_obs(rna, 'predicted_doublet', lambda x: x==False)
rna.layers['raw_counts'] = rna.X
sc.pp.normalize_total(rna, target_sum=1e4) #1e6 to do CPM normalization
sc.pp.log1p(rna)
rna.layers['lognorm_counts'] = rna.X
sc.pp.highly_variable_genes(rna, min_mean=0.02, max_mean=4, min_disp=0.5)
sc.pp.scale(rna, max_value=10)
rna.layers['scaled_counts'] = rna.X
rna.X = rna.layers['raw_counts'] # Restore original raw counts before Pearson residual normalization
analytic_pearson = sc.experimental.pp.normalize_pearson_residuals(rna, inplace=False)
rna.layers['analytic_pearson_residuals'] = csr_matrix(analytic_pearson['X'])
# ATAC
sc.pp.scrublet(atac)
doublets = sum(atac.obs.predicted_doublet)
mu.pp.filter_obs(atac, 'predicted_doublet', lambda x: x==False)
sc.pp.calculate_qc_metrics(atac, percent_top=None, log1p=False, inplace=True)
# Rename columns to make them fit to scATAC-seq (scanpy labels the metrics according to RNA-seq)
atac.obs.rename(
columns={
'n_genes_by_counts': 'n_features_per_cell',
'total_counts': 'total_fragment_counts'
},
inplace=True
)
atac.obs['log_total_fragment_counts'] = np.log10(atac.obs['total_fragment_counts'])
ac.tl.nucleosome_signal(atac)
nuc_signal_threshold = 2
atac.obs['nuc_signal_filter'] = [
'NS_FAIL' if ns > nuc_signal_threshold else 'NS_PASS'
for ns in atac.obs['nucleosome_signal']
]
ac.tl.get_gene_annotation_from_rna(mdata['rna']).head(3)
tss = ac.tl.tss_enrichment(mdata, n_tss=2000)
tss_threshold = 1.5
tss.obs["tss_filter"] = [
"TSS_FAIL" if score < tss_threshold else "TSS_PASS"
for score in atac.obs["tss_score"]
]
mu.pp.filter_obs(
atac,
"total_fragment_counts",
lambda x: (x >= 5000) & (x <= 45000),
)
mu.pp.filter_obs(
atac,
"n_features_per_cell",
lambda x: x >= 2200
)
mu.pp.filter_obs(
atac,
"tss_score",
lambda x: (x >= 2.0) & (x <= 20),
)
mu.pp.filter_obs(
atac,
"nucleosome_signal",
lambda x: x <= 2
)
mu.pp.filter_var(atac, "n_cells_by_counts", lambda x: x >= 10)
atac.layers["raw_counts"] = atac.X
ac.pp.tfidf(atac, scale_factor=1e4)
atac.layers['tf-idf_norm_counts'] = atac.X
atac.X = atac.layers["raw_counts"]
sc.pp.normalize_per_cell(atac, counts_per_cell_after=1e4)
sc.pp.log1p(atac)
atac.layers['lognorm_counts'] = atac.X
atac.X = atac.layers['tf-idf_norm_counts']
sc.pp.highly_variable_genes(atac, min_mean=0.05, max_mean=1.5, min_disp=0.5)
mu.pp.intersect_obs(mdata)
Thanks,
Leandro
Thanks, @Drito!
I can run this code on some test data without any issues.
Both data, on which that can be reproduced, and the full error log would be helpful in order to figure it out.
Hello,
I reinstalled muon again from GitHub and now everything works perfectly. Thanks for your help,
Drito
Hi , ran into the same issue and after reinstalling muon I am still seeing the error.
Here is the error log:
---------------------------------------------------------------------------
ValueError Traceback (most recent call last)
Cell In[24], line 1
----> 1 mu.pp.intersect_obs(mdata)
File ~/miniconda3/envs/single_cells/lib/python3.10/site-packages/muon/_core/preproc.py:653, in intersect_obs(mdata)
650 common_obs = reduce(np.intersect1d, [m.obs_names for m in mdata.mod.values()])
652 for mod in mdata.mod:
--> 653 filter_obs(mdata.mod[mod], common_obs)
655 mdata.update_obs()
657 return
File ~/miniconda3/envs/single_cells/lib/python3.10/site-packages/muon/_core/preproc.py:728, in filter_obs(data, var, func)
725 data._n_obs = data.obs.shape[0]
727 # Subset .obsm
--> 728 for k, v in data.obsm.items():
729 data.obsm[k] = v[obs_subset]
731 # Subset .obsp
File ~/miniconda3/envs/single_cells/lib/python3.10/site-packages/anndata/_core/aligned_mapping.py:423, in AlignedMappingProperty.__get__(self, obj, objtype)
421 return self # type: ignore
422 if not obj.is_view:
--> 423 return self.construct(obj, store=getattr(obj, f"_{self.name}"))
424 parent_anndata = obj._adata_ref
425 idxs = (obj._oidx, obj._vidx)
File ~/miniconda3/envs/single_cells/lib/python3.10/site-packages/anndata/_core/aligned_mapping.py:406, in AlignedMappingProperty.construct(self, obj, store)
404 if self.axis is None:
405 return self.cls(obj, store=store)
--> 406 return self.cls(obj, axis=self.axis, store=store)
File ~/miniconda3/envs/single_cells/lib/python3.10/site-packages/anndata/_core/aligned_mapping.py:295, in AxisArrays.__init__(self, parent, axis, store)
293 raise ValueError()
294 self._axis = axis
--> 295 super().__init__(parent, store=store)
File ~/miniconda3/envs/single_cells/lib/python3.10/site-packages/anndata/_core/aligned_mapping.py:208, in AlignedActual.__init__(self, parent, store)
206 self._data = store
207 for k, v in self._data.items():
--> 208 self._data[k] = self._validate_value(v, k)
File ~/miniconda3/envs/single_cells/lib/python3.10/site-packages/anndata/_core/aligned_mapping.py:277, in AxisArraysBase._validate_value(self, val, key)
275 msg = "Index.equals and pd.testing.assert_index_equal disagree"
276 raise AssertionError(msg)
--> 277 return super()._validate_value(val, key)
File ~/miniconda3/envs/single_cells/lib/python3.10/site-packages/anndata/_core/aligned_mapping.py:96, in AlignedMappingBase._validate_value(self, val, key)
90 dims = tuple(("obs", "var")[ax] for ax in self.axes)
91 msg = (
92 f"Value passed for key {key!r} is of incorrect shape. "
93 f"Values of {self.attrname} must match dimensions {dims} of parent. "
94 f"Value had shape {actual_shape} while it should have had {right_shape}."
95 )
---> 96 raise ValueError(msg)
98 name = f"{self.attrname.title().rstrip('s')} {key!r}"
99 return coerce_array(val, name=name, allow_df=self._allow_df)
ValueError: Value passed for key 'X_pca' is of incorrect shape. Values of obsm must match dimensions ('obs',) of parent. Value had shape (8460,) while it should have had (7969,).
Even if i just try running mdata I get the same error message but the log is different.
---------------------------------------------------------------------------
ValueError Traceback (most recent call last)
File ~/miniconda3/envs/single_cells/lib/python3.10/site-packages/IPython/core/formatters.py:770, in PlainTextFormatter.__call__(self, obj)
763 stream = StringIO()
764 printer = pretty.RepresentationPrinter(stream, self.verbose,
765 self.max_width, self.newline,
766 max_seq_length=self.max_seq_length,
767 singleton_pprinters=self.singleton_printers,
768 type_pprinters=self.type_printers,
769 deferred_pprinters=self.deferred_printers)
--> 770 printer.pretty(obj)
771 printer.flush()
772 return stream.getvalue()
File ~/miniconda3/envs/single_cells/lib/python3.10/site-packages/IPython/lib/pretty.py:419, in RepresentationPrinter.pretty(self, obj)
408 return meth(obj, self, cycle)
409 if (
410 cls is not object
411 # check if cls defines __repr__
(...)
417 and callable(_safe_getattr(cls, "__repr__", None))
418 ):
--> 419 return _repr_pprint(obj, self, cycle)
421 return _default_pprint(obj, self, cycle)
422 finally:
File ~/miniconda3/envs/single_cells/lib/python3.10/site-packages/IPython/lib/pretty.py:794, in _repr_pprint(obj, p, cycle)
792 """A pprint that just redirects to the normal repr function."""
793 # Find newlines and replace them with p.break_()
--> 794 output = repr(obj)
795 lines = output.splitlines()
796 with p.group():
File ~/miniconda3/envs/single_cells/lib/python3.10/site-packages/mudata/_core/mudata.py:2427, in MuData.__repr__(self)
2426 def __repr__(self) -> str:
-> 2427 return self._gen_repr(self.n_obs, self.n_vars, extensive=True)
File ~/miniconda3/envs/single_cells/lib/python3.10/site-packages/mudata/_core/mudata.py:2419, in MuData._gen_repr(self, n_obs, n_vars, extensive, nest_level)
2408 for attr in [
2409 "obs",
2410 "var",
(...)
2416 "varp",
2417 ]:
2418 try:
-> 2419 keys = getattr(v, attr).keys()
2420 if len(keys) > 0:
2421 descr += f"\n{mod_indent} {attr}:\t{str(list(keys))[1:-1]}"
File ~/miniconda3/envs/single_cells/lib/python3.10/site-packages/anndata/_core/aligned_mapping.py:423, in AlignedMappingProperty.__get__(self, obj, objtype)
421 return self # type: ignore
422 if not obj.is_view:
--> 423 return self.construct(obj, store=getattr(obj, f"_{self.name}"))
424 parent_anndata = obj._adata_ref
425 idxs = (obj._oidx, obj._vidx)
File ~/miniconda3/envs/single_cells/lib/python3.10/site-packages/anndata/_core/aligned_mapping.py:406, in AlignedMappingProperty.construct(self, obj, store)
404 if self.axis is None:
405 return self.cls(obj, store=store)
--> 406 return self.cls(obj, axis=self.axis, store=store)
File ~/miniconda3/envs/single_cells/lib/python3.10/site-packages/anndata/_core/aligned_mapping.py:295, in AxisArrays.__init__(self, parent, axis, store)
293 raise ValueError()
294 self._axis = axis
--> 295 super().__init__(parent, store=store)
File ~/miniconda3/envs/single_cells/lib/python3.10/site-packages/anndata/_core/aligned_mapping.py:208, in AlignedActual.__init__(self, parent, store)
206 self._data = store
207 for k, v in self._data.items():
--> 208 self._data[k] = self._validate_value(v, k)
File ~/miniconda3/envs/single_cells/lib/python3.10/site-packages/anndata/_core/aligned_mapping.py:277, in AxisArraysBase._validate_value(self, val, key)
275 msg = "Index.equals and pd.testing.assert_index_equal disagree"
276 raise AssertionError(msg)
--> 277 return super()._validate_value(val, key)
File ~/miniconda3/envs/single_cells/lib/python3.10/site-packages/anndata/_core/aligned_mapping.py:96, in AlignedMappingBase._validate_value(self, val, key)
90 dims = tuple(("obs", "var")[ax] for ax in self.axes)
91 msg = (
92 f"Value passed for key {key!r} is of incorrect shape. "
93 f"Values of {self.attrname} must match dimensions {dims} of parent. "
94 f"Value had shape {actual_shape} while it should have had {right_shape}."
95 )
---> 96 raise ValueError(msg)
98 name = f"{self.attrname.title().rstrip('s')} {key!r}"
99 return coerce_array(val, name=name, allow_df=self._allow_df)
ValueError: Value passed for key 'X_pca' is of incorrect shape. Values of obsm must match dimensions ('obs',) of parent. Value had shape (8460,) while it should have had (7969,).
---------------------------------------------------------------------------
ValueError Traceback (most recent call last)
File ~/miniconda3/envs/single_cells/lib/python3.10/site-packages/IPython/core/formatters.py:406, in BaseFormatter.__call__(self, obj)
404 method = get_real_method(obj, self.print_method)
405 if method is not None:
--> 406 return method()
407 return None
408 else:
File ~/miniconda3/envs/single_cells/lib/python3.10/site-packages/mudata/_core/mudata.py:2445, in MuData._repr_html_(self, expand)
2442 if OPTIONS["display_style"] == "text":
2443 from html import escape
-> 2445 return f"<pre>{escape(repr(self))}</pre>"
2447 if expand is None:
2448 expand = OPTIONS["display_html_expand"]
File ~/miniconda3/envs/single_cells/lib/python3.10/site-packages/mudata/_core/mudata.py:2427, in MuData.__repr__(self)
2426 def __repr__(self) -> str:
-> 2427 return self._gen_repr(self.n_obs, self.n_vars, extensive=True)
File ~/miniconda3/envs/single_cells/lib/python3.10/site-packages/mudata/_core/mudata.py:2419, in MuData._gen_repr(self, n_obs, n_vars, extensive, nest_level)
2408 for attr in [
2409 "obs",
2410 "var",
(...)
2416 "varp",
2417 ]:
2418 try:
-> 2419 keys = getattr(v, attr).keys()
2420 if len(keys) > 0:
2421 descr += f"\n{mod_indent} {attr}:\t{str(list(keys))[1:-1]}"
File ~/miniconda3/envs/single_cells/lib/python3.10/site-packages/anndata/_core/aligned_mapping.py:423, in AlignedMappingProperty.__get__(self, obj, objtype)
421 return self # type: ignore
422 if not obj.is_view:
--> 423 return self.construct(obj, store=getattr(obj, f"_{self.name}"))
424 parent_anndata = obj._adata_ref
425 idxs = (obj._oidx, obj._vidx)
File ~/miniconda3/envs/single_cells/lib/python3.10/site-packages/anndata/_core/aligned_mapping.py:406, in AlignedMappingProperty.construct(self, obj, store)
404 if self.axis is None:
405 return self.cls(obj, store=store)
--> 406 return self.cls(obj, axis=self.axis, store=store)
File ~/miniconda3/envs/single_cells/lib/python3.10/site-packages/anndata/_core/aligned_mapping.py:295, in AxisArrays.__init__(self, parent, axis, store)
293 raise ValueError()
294 self._axis = axis
--> 295 super().__init__(parent, store=store)
File ~/miniconda3/envs/single_cells/lib/python3.10/site-packages/anndata/_core/aligned_mapping.py:208, in AlignedActual.__init__(self, parent, store)
206 self._data = store
207 for k, v in self._data.items():
--> 208 self._data[k] = self._validate_value(v, k)
File ~/miniconda3/envs/single_cells/lib/python3.10/site-packages/anndata/_core/aligned_mapping.py:277, in AxisArraysBase._validate_value(self, val, key)
275 msg = "Index.equals and pd.testing.assert_index_equal disagree"
276 raise AssertionError(msg)
--> 277 return super()._validate_value(val, key)
File ~/miniconda3/envs/single_cells/lib/python3.10/site-packages/anndata/_core/aligned_mapping.py:96, in AlignedMappingBase._validate_value(self, val, key)
90 dims = tuple(("obs", "var")[ax] for ax in self.axes)
91 msg = (
92 f"Value passed for key {key!r} is of incorrect shape. "
93 f"Values of {self.attrname} must match dimensions {dims} of parent. "
94 f"Value had shape {actual_shape} while it should have had {right_shape}."
95 )
---> 96 raise ValueError(msg)
98 name = f"{self.attrname.title().rstrip('s')} {key!r}"
99 return coerce_array(val, name=name, allow_df=self._allow_df)
ValueError: Value passed for key 'X_pca' is of incorrect shape. Values of obsm must match dimensions ('obs',) of parent. Value had shape (8460,) while it should have had (7969,).