I think it's not moving by using XML, Because It's commenting in **module-catalog/templates/product/list.phtml** file.

    <?php echo $block->getToolbarHtml() ?>

Also it's define in module-catalog/view/frontend/layout/catalog_category_view.xml **category.products.list** block

    <referenceContainer name="content">
                <block class="Magento\Catalog\Block\Category\View" name="category.products" template="Magento_Catalog::category/products.phtml">
                    <block class="Magento\Catalog\Block\Product\ListProduct" name="category.products.list" as="product_list" template="Magento_Catalog::product/list.phtml">
                        <container name="category.product.list.additional" as="additional" />
                        <block class="Magento\Framework\View\Element\RendererList" name="category.product.type.details.renderers" as="details.renderers">
                            <block class="Magento\Framework\View\Element\Template" name="category.product.type.details.renderers.default" as="default"/>
                        </block>
                        <block class="Magento\Catalog\Block\Product\ProductList\Item\Container" name="category.product.addto" as="addto">
                            <block class="Magento\Catalog\Block\Product\ProductList\Item\AddTo\Compare"
                                   name="category.product.addto.compare" as="compare"
                                   template="Magento_Catalog::product/list/addto/compare.phtml"/>
                        </block>
                        <block class="Magento\Catalog\Block\Product\ProductList\Toolbar" name="product_list_toolbar" template="Magento_Catalog::product/list/toolbar.phtml">
                            <block class="Magento\Theme\Block\Html\Pager" name="product_list_toolbar_pager"/>
                        </block>
                        <action method="setToolbarBlockName">
                            <argument name="name" xsi:type="string">product_list_toolbar</argument>
                        </action>
                    </block>
                </block>
            </referenceContainer>