remove astropy and specutils from downstream

astropy was included here because of astropy.io.misc.asdf
which is deprecated and uses the legacy extension api (which
will be removed during this development cycle). The remaining
usage of asdf in astropy is through asdf-astropy.

specutils includes a legacy extension that may be moved to
asdf-astropy. Per advice from WilliamJamieson specutils is being
removed from downstream testing.

.github/workflows/downstream.yml

@@ -40,8 +40,6 @@ jobs:
       # Any env name which does not start with `pyXY` will use this Python version.
       default_python: '3.10'
       envs: |
-        - linux: astropy
-        - linux: specutils
         - linux: asdf-astropy
 
   stsci:

tox.ini

@@ -114,22 +114,6 @@ commands_pre =
 commands =
     pytest asdf-coordinates-schemas
 
-[testenv:astropy]
-change_dir = {env_tmp_dir}
-allowlist_externals =
-    git
-    bash
-extras =
-commands_pre =
-    bash -c "pip freeze -q | grep 'asdf @' > {env_tmp_dir}/requirements.txt"
-    git clone https://github.com/astropy/astropy.git
-    pip install -e astropy[test]
-    pip install -r {env_tmp_dir}/requirements.txt
-    pip freeze
-commands=
-    pytest astropy/astropy/io/misc/asdf --open-files --run-slow --remote-data \
-        -W "ignore::asdf.exceptions.AsdfDeprecationWarning:asdf.tests.helpers"
-
 [testenv:asdf-astropy]
 change_dir = {env_tmp_dir}
 allowlist_externals =
@@ -143,24 +127,7 @@ commands_pre =
     pip install -r {env_tmp_dir}/requirements.txt
     pip freeze
 commands =
-    pytest asdf-astropy \
-        -W "ignore::asdf.exceptions.AsdfDeprecationWarning:asdf.tests.helpers"
-
-[testenv:specutils]
-change_dir = {env_tmp_dir}
-allowlist_externals =
-    git
-    bash
-extras =
-commands_pre =
-    bash -c "pip freeze -q | grep 'asdf @' > {env_tmp_dir}/requirements.txt"
-    git clone https://github.com/astropy/specutils.git
-    pip install -e specutils[test]
-    pip install -r {env_tmp_dir}/requirements.txt
-    pip freeze
-commands =
-    pytest specutils \
-        -W "ignore::asdf.exceptions.AsdfDeprecationWarning:asdf.tests.helpers"
+    pytest asdf-astropy
 
 [testenv:gwcs]
 change_dir = {env_tmp_dir}