quick workaround can be add a condition in theme templates file Magento_Catalog/templates/product/list/toolbar/sorter.phtml
like
if $key != 'position' then don't add in drop down options :
<select id="sorter" data-role="sorter" class="sorter-options">
<?php foreach ($block->getAvailableOrders() as $_key => $_order): ?>
<?php if ($_key != 'position') : // if key sort by position remove it?>
<option value="<?= /* @escapeNotVerified */ $_key ?>"
<?php if ($block->isOrderCurrent($_key)): ?>
selected="selected"
<?php endif; ?>
>
<?= $block->escapeHtml(__($_order)) ?>
</option>
<?php endif; ?>
<?php endforeach; ?>
</select>