You cannot only remove js validation, there is backend address validation which hardcode to check some fields in address must not be empty.
app/code/core/Mage/Customer/Model/Address/Abstract.php
protected function _basicCheck()
{
if (!Zend_Validate::is($this->getFirstname(), 'NotEmpty')) {
$this->addError(Mage::helper('customer')->__('Please enter the first name.'));
}
if (!Zend_Validate::is($this->getLastname(), 'NotEmpty')) {
$this->addError(Mage::helper('customer')->__('Please enter the last name.'));
}
if (!Zend_Validate::is($this->getStreet(1), 'NotEmpty')) {
$this->addError(Mage::helper('customer')->__('Please enter the street.'));
}
if (!Zend_Validate::is($this->getCity(), 'NotEmpty')) {
$this->addError(Mage::helper('customer')->__('Please enter the city.'));
}