How to set a field editable only for a group?

I found it!

First, the field has to be defined as readonly for everyone.

<xpath expr="//field[@name='employee_id']" position="replace">
     <field name="employee_id" attrs="{'readonly':True}"/>
</xpath>

Then, we inherit the first custom view

<field name="inherit_id" ref="hr_attendance_extend.view_employee_readonly_custom"/>

Finally, we delete the readonly restriction for the administrators group (group_hr_manager)

<field name="groups_id" eval="[(6, 0, [ref('base.group_hr_manager')])]"/>
<field name="arch" type="xml">
    <xpath expr="//field[@name='employee_id']" position="attributes">
        <attribute name="readonly">False</attribute>
    </xpath>
</field>

Here is the final code:
<?xml version="1.0" encoding="utf-8"?>
<openerp>
    <data>
    <record id="view_employee_readonly_custom" model="ir.ui.view">
        <field name="name">hr.attendance.form</field>
        <field name="model">hr.attendance</field>
        <field name="inherit_id" ref="hr_attendance.view_attendance_form"/>
        <field name="arch" type="xml">
            <xpath expr="//field[@name='employee_id']" position="replace">
                <field name="employee_id" attrs="{'readonly':True}"/>
            </xpath>
        </field>
    </record>

    <record id="view_employee_readonly" model="ir.ui.view">
        <field name="name">hr.attendance.form</field>
        <field name="model">hr.attendance</field>
        <field name="inherit_id" ref="hr_attendance_extend.view_employee_readonly_custom" />
        <field name="groups_id" eval="[(6, 0, [ref('base.group_hr_manager')])]"/>
        <field name="arch" type="xml">
            <xpath expr="//field[@name='employee_id']" position="attributes">
                <attribute name="readonly">False</attribute>
            </xpath>
        </field>
    </record>
    </data>
</openerp>